lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_table_af_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t table_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[table_id];
2107         struct table_runtime *table = &t->tables[table_id];
2108         struct table_statistics *stats = &p->table_stats[table_id];
2109         uint64_t action_id, n_pkts_hit, n_pkts_action;
2110         uint8_t *action_data;
2111         action_func_t action_func;
2112         int done, hit;
2113
2114         /* Table. */
2115         done = table->func(ts->obj,
2116                            table->mailbox,
2117                            table->key,
2118                            &action_id,
2119                            &action_data,
2120                            &hit);
2121         if (!done) {
2122                 /* Thread. */
2123                 TRACE("[Thread %2u] table %u (not finalized)\n",
2124                       p->thread_id,
2125                       table_id);
2126
2127                 thread_yield(p);
2128                 return;
2129         }
2130
2131         action_id = hit ? action_id : ts->default_action_id;
2132         action_data = hit ? action_data : ts->default_action_data;
2133         action_func = p->action_funcs[action_id];
2134         n_pkts_hit = stats->n_pkts_hit[hit];
2135         n_pkts_action = stats->n_pkts_action[action_id];
2136
2137         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2138               p->thread_id,
2139               table_id,
2140               hit ? "hit" : "miss",
2141               (uint32_t)action_id);
2142
2143         t->action_id = action_id;
2144         t->structs[0] = action_data;
2145         t->hit = hit;
2146         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2147         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2148
2149         /* Thread. */
2150         thread_ip_inc(p);
2151
2152         /* Action. */
2153         action_func(p);
2154 }
2155
2156 static inline void
2157 instr_selector_exec(struct rte_swx_pipeline *p)
2158 {
2159         struct thread *t = &p->threads[p->thread_id];
2160         struct instruction *ip = t->ip;
2161         uint32_t selector_id = ip->table.table_id;
2162         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2163         struct selector_runtime *selector = &t->selectors[selector_id];
2164         struct selector_statistics *stats = &p->selector_stats[selector_id];
2165         uint64_t n_pkts = stats->n_pkts;
2166         int done;
2167
2168         /* Table. */
2169         done = rte_swx_table_selector_select(ts->obj,
2170                            selector->mailbox,
2171                            selector->group_id_buffer,
2172                            selector->selector_buffer,
2173                            selector->member_id_buffer);
2174         if (!done) {
2175                 /* Thread. */
2176                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2177                       p->thread_id,
2178                       selector_id);
2179
2180                 thread_yield(p);
2181                 return;
2182         }
2183
2184
2185         TRACE("[Thread %2u] selector %u\n",
2186               p->thread_id,
2187               selector_id);
2188
2189         stats->n_pkts = n_pkts + 1;
2190
2191         /* Thread. */
2192         thread_ip_inc(p);
2193 }
2194
2195 static inline void
2196 instr_learner_exec(struct rte_swx_pipeline *p)
2197 {
2198         struct thread *t = &p->threads[p->thread_id];
2199         struct instruction *ip = t->ip;
2200         uint32_t learner_id = ip->table.table_id;
2201         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2202                 p->n_selectors + learner_id];
2203         struct learner_runtime *l = &t->learners[learner_id];
2204         struct learner_statistics *stats = &p->learner_stats[learner_id];
2205         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2206         uint8_t *action_data;
2207         int done, hit;
2208
2209         /* Table. */
2210         time = rte_get_tsc_cycles();
2211
2212         done = rte_swx_table_learner_lookup(ts->obj,
2213                                             l->mailbox,
2214                                             time,
2215                                             l->key,
2216                                             &action_id,
2217                                             &action_data,
2218                                             &hit);
2219         if (!done) {
2220                 /* Thread. */
2221                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2222                       p->thread_id,
2223                       learner_id);
2224
2225                 thread_yield(p);
2226                 return;
2227         }
2228
2229         action_id = hit ? action_id : ts->default_action_id;
2230         action_data = hit ? action_data : ts->default_action_data;
2231         n_pkts_hit = stats->n_pkts_hit[hit];
2232         n_pkts_action = stats->n_pkts_action[action_id];
2233
2234         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2235               p->thread_id,
2236               learner_id,
2237               hit ? "hit" : "miss",
2238               (uint32_t)action_id);
2239
2240         t->action_id = action_id;
2241         t->structs[0] = action_data;
2242         t->hit = hit;
2243         t->learner_id = learner_id;
2244         t->time = time;
2245         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2246         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2247
2248         /* Thread. */
2249         thread_ip_action_call(p, t, action_id);
2250 }
2251
2252 static inline void
2253 instr_learner_af_exec(struct rte_swx_pipeline *p)
2254 {
2255         struct thread *t = &p->threads[p->thread_id];
2256         struct instruction *ip = t->ip;
2257         uint32_t learner_id = ip->table.table_id;
2258         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2259                 p->n_selectors + learner_id];
2260         struct learner_runtime *l = &t->learners[learner_id];
2261         struct learner_statistics *stats = &p->learner_stats[learner_id];
2262         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2263         uint8_t *action_data;
2264         action_func_t action_func;
2265         int done, hit;
2266
2267         /* Table. */
2268         time = rte_get_tsc_cycles();
2269
2270         done = rte_swx_table_learner_lookup(ts->obj,
2271                                             l->mailbox,
2272                                             time,
2273                                             l->key,
2274                                             &action_id,
2275                                             &action_data,
2276                                             &hit);
2277         if (!done) {
2278                 /* Thread. */
2279                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2280                       p->thread_id,
2281                       learner_id);
2282
2283                 thread_yield(p);
2284                 return;
2285         }
2286
2287         action_id = hit ? action_id : ts->default_action_id;
2288         action_data = hit ? action_data : ts->default_action_data;
2289         action_func = p->action_funcs[action_id];
2290         n_pkts_hit = stats->n_pkts_hit[hit];
2291         n_pkts_action = stats->n_pkts_action[action_id];
2292
2293         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2294               p->thread_id,
2295               learner_id,
2296               hit ? "hit" : "miss",
2297               (uint32_t)action_id);
2298
2299         t->action_id = action_id;
2300         t->structs[0] = action_data;
2301         t->hit = hit;
2302         t->learner_id = learner_id;
2303         t->time = time;
2304         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2305         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2306
2307         /* Thread. */
2308         thread_ip_action_call(p, t, action_id);
2309
2310         /* Action */
2311         action_func(p);
2312 }
2313
2314 /*
2315  * learn.
2316  */
2317 static struct action *
2318 action_find(struct rte_swx_pipeline *p, const char *name);
2319
2320 static int
2321 action_has_nbo_args(struct action *a);
2322
2323 static int
2324 instr_learn_translate(struct rte_swx_pipeline *p,
2325                       struct action *action,
2326                       char **tokens,
2327                       int n_tokens,
2328                       struct instruction *instr,
2329                       struct instruction_data *data __rte_unused)
2330 {
2331         struct action *a;
2332
2333         CHECK(action, EINVAL);
2334         CHECK(n_tokens == 2, EINVAL);
2335
2336         a = action_find(p, tokens[1]);
2337         CHECK(a, EINVAL);
2338         CHECK(!action_has_nbo_args(a), EINVAL);
2339
2340         instr->type = INSTR_LEARNER_LEARN;
2341         instr->learn.action_id = a->id;
2342
2343         return 0;
2344 }
2345
2346 static inline void
2347 instr_learn_exec(struct rte_swx_pipeline *p)
2348 {
2349         struct thread *t = &p->threads[p->thread_id];
2350         struct instruction *ip = t->ip;
2351
2352         __instr_learn_exec(p, t, ip);
2353
2354         /* Thread. */
2355         thread_ip_inc(p);
2356 }
2357
2358 /*
2359  * forget.
2360  */
2361 static int
2362 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2363                        struct action *action,
2364                        char **tokens __rte_unused,
2365                        int n_tokens,
2366                        struct instruction *instr,
2367                        struct instruction_data *data __rte_unused)
2368 {
2369         CHECK(action, EINVAL);
2370         CHECK(n_tokens == 1, EINVAL);
2371
2372         instr->type = INSTR_LEARNER_FORGET;
2373
2374         return 0;
2375 }
2376
2377 static inline void
2378 instr_forget_exec(struct rte_swx_pipeline *p)
2379 {
2380         struct thread *t = &p->threads[p->thread_id];
2381         struct instruction *ip = t->ip;
2382
2383         __instr_forget_exec(p, t, ip);
2384
2385         /* Thread. */
2386         thread_ip_inc(p);
2387 }
2388
2389 /*
2390  * extern.
2391  */
2392 static int
2393 instr_extern_translate(struct rte_swx_pipeline *p,
2394                        struct action *action __rte_unused,
2395                        char **tokens,
2396                        int n_tokens,
2397                        struct instruction *instr,
2398                        struct instruction_data *data __rte_unused)
2399 {
2400         char *token = tokens[1];
2401
2402         CHECK(n_tokens == 2, EINVAL);
2403
2404         if (token[0] == 'e') {
2405                 struct extern_obj *obj;
2406                 struct extern_type_member_func *func;
2407
2408                 func = extern_obj_member_func_parse(p, token, &obj);
2409                 CHECK(func, EINVAL);
2410
2411                 instr->type = INSTR_EXTERN_OBJ;
2412                 instr->ext_obj.ext_obj_id = obj->id;
2413                 instr->ext_obj.func_id = func->id;
2414
2415                 return 0;
2416         }
2417
2418         if (token[0] == 'f') {
2419                 struct extern_func *func;
2420
2421                 func = extern_func_parse(p, token);
2422                 CHECK(func, EINVAL);
2423
2424                 instr->type = INSTR_EXTERN_FUNC;
2425                 instr->ext_func.ext_func_id = func->id;
2426
2427                 return 0;
2428         }
2429
2430         CHECK(0, EINVAL);
2431 }
2432
2433 static inline void
2434 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2435 {
2436         struct thread *t = &p->threads[p->thread_id];
2437         struct instruction *ip = t->ip;
2438         uint32_t done;
2439
2440         /* Extern object member function execute. */
2441         done = __instr_extern_obj_exec(p, t, ip);
2442
2443         /* Thread. */
2444         thread_ip_inc_cond(t, done);
2445         thread_yield_cond(p, done ^ 1);
2446 }
2447
2448 static inline void
2449 instr_extern_func_exec(struct rte_swx_pipeline *p)
2450 {
2451         struct thread *t = &p->threads[p->thread_id];
2452         struct instruction *ip = t->ip;
2453         uint32_t done;
2454
2455         /* Extern function execute. */
2456         done = __instr_extern_func_exec(p, t, ip);
2457
2458         /* Thread. */
2459         thread_ip_inc_cond(t, done);
2460         thread_yield_cond(p, done ^ 1);
2461 }
2462
2463 /*
2464  * mov.
2465  */
2466 static int
2467 instr_mov_translate(struct rte_swx_pipeline *p,
2468                     struct action *action,
2469                     char **tokens,
2470                     int n_tokens,
2471                     struct instruction *instr,
2472                     struct instruction_data *data __rte_unused)
2473 {
2474         char *dst = tokens[1], *src = tokens[2];
2475         struct field *fdst, *fsrc;
2476         uint64_t src_val;
2477         uint32_t dst_struct_id = 0, src_struct_id = 0;
2478
2479         CHECK(n_tokens == 3, EINVAL);
2480
2481         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2482         CHECK(fdst, EINVAL);
2483         CHECK(!fdst->var_size, EINVAL);
2484
2485         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2486         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2487         if (fsrc) {
2488                 CHECK(!fsrc->var_size, EINVAL);
2489
2490                 instr->type = INSTR_MOV;
2491                 if (dst[0] != 'h' && src[0] == 'h')
2492                         instr->type = INSTR_MOV_MH;
2493                 if (dst[0] == 'h' && src[0] != 'h')
2494                         instr->type = INSTR_MOV_HM;
2495                 if (dst[0] == 'h' && src[0] == 'h')
2496                         instr->type = INSTR_MOV_HH;
2497
2498                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2499                 instr->mov.dst.n_bits = fdst->n_bits;
2500                 instr->mov.dst.offset = fdst->offset / 8;
2501                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2502                 instr->mov.src.n_bits = fsrc->n_bits;
2503                 instr->mov.src.offset = fsrc->offset / 8;
2504                 return 0;
2505         }
2506
2507         /* MOV_I. */
2508         src_val = strtoull(src, &src, 0);
2509         CHECK(!src[0], EINVAL);
2510
2511         if (dst[0] == 'h')
2512                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2513
2514         instr->type = INSTR_MOV_I;
2515         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2516         instr->mov.dst.n_bits = fdst->n_bits;
2517         instr->mov.dst.offset = fdst->offset / 8;
2518         instr->mov.src_val = src_val;
2519         return 0;
2520 }
2521
2522 static inline void
2523 instr_mov_exec(struct rte_swx_pipeline *p)
2524 {
2525         struct thread *t = &p->threads[p->thread_id];
2526         struct instruction *ip = t->ip;
2527
2528         __instr_mov_exec(p, t, ip);
2529
2530         /* Thread. */
2531         thread_ip_inc(p);
2532 }
2533
2534 static inline void
2535 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2536 {
2537         struct thread *t = &p->threads[p->thread_id];
2538         struct instruction *ip = t->ip;
2539
2540         __instr_mov_mh_exec(p, t, ip);
2541
2542         /* Thread. */
2543         thread_ip_inc(p);
2544 }
2545
2546 static inline void
2547 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2548 {
2549         struct thread *t = &p->threads[p->thread_id];
2550         struct instruction *ip = t->ip;
2551
2552         __instr_mov_hm_exec(p, t, ip);
2553
2554         /* Thread. */
2555         thread_ip_inc(p);
2556 }
2557
2558 static inline void
2559 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2560 {
2561         struct thread *t = &p->threads[p->thread_id];
2562         struct instruction *ip = t->ip;
2563
2564         __instr_mov_hh_exec(p, t, ip);
2565
2566         /* Thread. */
2567         thread_ip_inc(p);
2568 }
2569
2570 static inline void
2571 instr_mov_i_exec(struct rte_swx_pipeline *p)
2572 {
2573         struct thread *t = &p->threads[p->thread_id];
2574         struct instruction *ip = t->ip;
2575
2576         __instr_mov_i_exec(p, t, ip);
2577
2578         /* Thread. */
2579         thread_ip_inc(p);
2580 }
2581
2582 /*
2583  * dma.
2584  */
2585 static inline void
2586 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2587 {
2588         struct thread *t = &p->threads[p->thread_id];
2589         struct instruction *ip = t->ip;
2590
2591         __instr_dma_ht_exec(p, t, ip);
2592
2593         /* Thread. */
2594         thread_ip_inc(p);
2595 }
2596
2597 static inline void
2598 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2599 {
2600         struct thread *t = &p->threads[p->thread_id];
2601         struct instruction *ip = t->ip;
2602
2603         __instr_dma_ht2_exec(p, t, ip);
2604
2605         /* Thread. */
2606         thread_ip_inc(p);
2607 }
2608
2609 static inline void
2610 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2611 {
2612         struct thread *t = &p->threads[p->thread_id];
2613         struct instruction *ip = t->ip;
2614
2615         __instr_dma_ht3_exec(p, t, ip);
2616
2617         /* Thread. */
2618         thread_ip_inc(p);
2619 }
2620
2621 static inline void
2622 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2623 {
2624         struct thread *t = &p->threads[p->thread_id];
2625         struct instruction *ip = t->ip;
2626
2627         __instr_dma_ht4_exec(p, t, ip);
2628
2629         /* Thread. */
2630         thread_ip_inc(p);
2631 }
2632
2633 static inline void
2634 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2635 {
2636         struct thread *t = &p->threads[p->thread_id];
2637         struct instruction *ip = t->ip;
2638
2639         __instr_dma_ht5_exec(p, t, ip);
2640
2641         /* Thread. */
2642         thread_ip_inc(p);
2643 }
2644
2645 static inline void
2646 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2647 {
2648         struct thread *t = &p->threads[p->thread_id];
2649         struct instruction *ip = t->ip;
2650
2651         __instr_dma_ht6_exec(p, t, ip);
2652
2653         /* Thread. */
2654         thread_ip_inc(p);
2655 }
2656
2657 static inline void
2658 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2659 {
2660         struct thread *t = &p->threads[p->thread_id];
2661         struct instruction *ip = t->ip;
2662
2663         __instr_dma_ht7_exec(p, t, ip);
2664
2665         /* Thread. */
2666         thread_ip_inc(p);
2667 }
2668
2669 static inline void
2670 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2671 {
2672         struct thread *t = &p->threads[p->thread_id];
2673         struct instruction *ip = t->ip;
2674
2675         __instr_dma_ht8_exec(p, t, ip);
2676
2677         /* Thread. */
2678         thread_ip_inc(p);
2679 }
2680
2681 /*
2682  * alu.
2683  */
2684 static int
2685 instr_alu_add_translate(struct rte_swx_pipeline *p,
2686                         struct action *action,
2687                         char **tokens,
2688                         int n_tokens,
2689                         struct instruction *instr,
2690                         struct instruction_data *data __rte_unused)
2691 {
2692         char *dst = tokens[1], *src = tokens[2];
2693         struct field *fdst, *fsrc;
2694         uint64_t src_val;
2695         uint32_t dst_struct_id = 0, src_struct_id = 0;
2696
2697         CHECK(n_tokens == 3, EINVAL);
2698
2699         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2700         CHECK(fdst, EINVAL);
2701         CHECK(!fdst->var_size, EINVAL);
2702
2703         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2704         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2705         if (fsrc) {
2706                 CHECK(!fsrc->var_size, EINVAL);
2707
2708                 instr->type = INSTR_ALU_ADD;
2709                 if (dst[0] == 'h' && src[0] != 'h')
2710                         instr->type = INSTR_ALU_ADD_HM;
2711                 if (dst[0] != 'h' && src[0] == 'h')
2712                         instr->type = INSTR_ALU_ADD_MH;
2713                 if (dst[0] == 'h' && src[0] == 'h')
2714                         instr->type = INSTR_ALU_ADD_HH;
2715
2716                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2717                 instr->alu.dst.n_bits = fdst->n_bits;
2718                 instr->alu.dst.offset = fdst->offset / 8;
2719                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2720                 instr->alu.src.n_bits = fsrc->n_bits;
2721                 instr->alu.src.offset = fsrc->offset / 8;
2722                 return 0;
2723         }
2724
2725         /* ADD_MI, ADD_HI. */
2726         src_val = strtoull(src, &src, 0);
2727         CHECK(!src[0], EINVAL);
2728
2729         instr->type = INSTR_ALU_ADD_MI;
2730         if (dst[0] == 'h')
2731                 instr->type = INSTR_ALU_ADD_HI;
2732
2733         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2734         instr->alu.dst.n_bits = fdst->n_bits;
2735         instr->alu.dst.offset = fdst->offset / 8;
2736         instr->alu.src_val = src_val;
2737         return 0;
2738 }
2739
2740 static int
2741 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2742                         struct action *action,
2743                         char **tokens,
2744                         int n_tokens,
2745                         struct instruction *instr,
2746                         struct instruction_data *data __rte_unused)
2747 {
2748         char *dst = tokens[1], *src = tokens[2];
2749         struct field *fdst, *fsrc;
2750         uint64_t src_val;
2751         uint32_t dst_struct_id = 0, src_struct_id = 0;
2752
2753         CHECK(n_tokens == 3, EINVAL);
2754
2755         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2756         CHECK(fdst, EINVAL);
2757         CHECK(!fdst->var_size, EINVAL);
2758
2759         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2760         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2761         if (fsrc) {
2762                 CHECK(!fsrc->var_size, EINVAL);
2763
2764                 instr->type = INSTR_ALU_SUB;
2765                 if (dst[0] == 'h' && src[0] != 'h')
2766                         instr->type = INSTR_ALU_SUB_HM;
2767                 if (dst[0] != 'h' && src[0] == 'h')
2768                         instr->type = INSTR_ALU_SUB_MH;
2769                 if (dst[0] == 'h' && src[0] == 'h')
2770                         instr->type = INSTR_ALU_SUB_HH;
2771
2772                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2773                 instr->alu.dst.n_bits = fdst->n_bits;
2774                 instr->alu.dst.offset = fdst->offset / 8;
2775                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2776                 instr->alu.src.n_bits = fsrc->n_bits;
2777                 instr->alu.src.offset = fsrc->offset / 8;
2778                 return 0;
2779         }
2780
2781         /* SUB_MI, SUB_HI. */
2782         src_val = strtoull(src, &src, 0);
2783         CHECK(!src[0], EINVAL);
2784
2785         instr->type = INSTR_ALU_SUB_MI;
2786         if (dst[0] == 'h')
2787                 instr->type = INSTR_ALU_SUB_HI;
2788
2789         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2790         instr->alu.dst.n_bits = fdst->n_bits;
2791         instr->alu.dst.offset = fdst->offset / 8;
2792         instr->alu.src_val = src_val;
2793         return 0;
2794 }
2795
2796 static int
2797 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2798                           struct action *action __rte_unused,
2799                           char **tokens,
2800                           int n_tokens,
2801                           struct instruction *instr,
2802                           struct instruction_data *data __rte_unused)
2803 {
2804         char *dst = tokens[1], *src = tokens[2];
2805         struct header *hdst, *hsrc;
2806         struct field *fdst, *fsrc;
2807
2808         CHECK(n_tokens == 3, EINVAL);
2809
2810         fdst = header_field_parse(p, dst, &hdst);
2811         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2812         CHECK(!fdst->var_size, EINVAL);
2813
2814         /* CKADD_FIELD. */
2815         fsrc = header_field_parse(p, src, &hsrc);
2816         if (fsrc) {
2817                 CHECK(!fsrc->var_size, EINVAL);
2818
2819                 instr->type = INSTR_ALU_CKADD_FIELD;
2820                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2821                 instr->alu.dst.n_bits = fdst->n_bits;
2822                 instr->alu.dst.offset = fdst->offset / 8;
2823                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2824                 instr->alu.src.n_bits = fsrc->n_bits;
2825                 instr->alu.src.offset = fsrc->offset / 8;
2826                 return 0;
2827         }
2828
2829         /* CKADD_STRUCT, CKADD_STRUCT20. */
2830         hsrc = header_parse(p, src);
2831         CHECK(hsrc, EINVAL);
2832         CHECK(!hsrc->st->var_size, EINVAL);
2833
2834         instr->type = INSTR_ALU_CKADD_STRUCT;
2835         if ((hsrc->st->n_bits / 8) == 20)
2836                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2837
2838         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2839         instr->alu.dst.n_bits = fdst->n_bits;
2840         instr->alu.dst.offset = fdst->offset / 8;
2841         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2842         instr->alu.src.n_bits = hsrc->st->n_bits;
2843         instr->alu.src.offset = 0; /* Unused. */
2844         return 0;
2845 }
2846
2847 static int
2848 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2849                           struct action *action __rte_unused,
2850                           char **tokens,
2851                           int n_tokens,
2852                           struct instruction *instr,
2853                           struct instruction_data *data __rte_unused)
2854 {
2855         char *dst = tokens[1], *src = tokens[2];
2856         struct header *hdst, *hsrc;
2857         struct field *fdst, *fsrc;
2858
2859         CHECK(n_tokens == 3, EINVAL);
2860
2861         fdst = header_field_parse(p, dst, &hdst);
2862         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2863         CHECK(!fdst->var_size, EINVAL);
2864
2865         fsrc = header_field_parse(p, src, &hsrc);
2866         CHECK(fsrc, EINVAL);
2867         CHECK(!fsrc->var_size, EINVAL);
2868
2869         instr->type = INSTR_ALU_CKSUB_FIELD;
2870         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2871         instr->alu.dst.n_bits = fdst->n_bits;
2872         instr->alu.dst.offset = fdst->offset / 8;
2873         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2874         instr->alu.src.n_bits = fsrc->n_bits;
2875         instr->alu.src.offset = fsrc->offset / 8;
2876         return 0;
2877 }
2878
2879 static int
2880 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2881                         struct action *action,
2882                         char **tokens,
2883                         int n_tokens,
2884                         struct instruction *instr,
2885                         struct instruction_data *data __rte_unused)
2886 {
2887         char *dst = tokens[1], *src = tokens[2];
2888         struct field *fdst, *fsrc;
2889         uint64_t src_val;
2890         uint32_t dst_struct_id = 0, src_struct_id = 0;
2891
2892         CHECK(n_tokens == 3, EINVAL);
2893
2894         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2895         CHECK(fdst, EINVAL);
2896         CHECK(!fdst->var_size, EINVAL);
2897
2898         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2899         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2900         if (fsrc) {
2901                 CHECK(!fsrc->var_size, EINVAL);
2902
2903                 instr->type = INSTR_ALU_SHL;
2904                 if (dst[0] == 'h' && src[0] != 'h')
2905                         instr->type = INSTR_ALU_SHL_HM;
2906                 if (dst[0] != 'h' && src[0] == 'h')
2907                         instr->type = INSTR_ALU_SHL_MH;
2908                 if (dst[0] == 'h' && src[0] == 'h')
2909                         instr->type = INSTR_ALU_SHL_HH;
2910
2911                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2912                 instr->alu.dst.n_bits = fdst->n_bits;
2913                 instr->alu.dst.offset = fdst->offset / 8;
2914                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2915                 instr->alu.src.n_bits = fsrc->n_bits;
2916                 instr->alu.src.offset = fsrc->offset / 8;
2917                 return 0;
2918         }
2919
2920         /* SHL_MI, SHL_HI. */
2921         src_val = strtoull(src, &src, 0);
2922         CHECK(!src[0], EINVAL);
2923
2924         instr->type = INSTR_ALU_SHL_MI;
2925         if (dst[0] == 'h')
2926                 instr->type = INSTR_ALU_SHL_HI;
2927
2928         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2929         instr->alu.dst.n_bits = fdst->n_bits;
2930         instr->alu.dst.offset = fdst->offset / 8;
2931         instr->alu.src_val = src_val;
2932         return 0;
2933 }
2934
2935 static int
2936 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2937                         struct action *action,
2938                         char **tokens,
2939                         int n_tokens,
2940                         struct instruction *instr,
2941                         struct instruction_data *data __rte_unused)
2942 {
2943         char *dst = tokens[1], *src = tokens[2];
2944         struct field *fdst, *fsrc;
2945         uint64_t src_val;
2946         uint32_t dst_struct_id = 0, src_struct_id = 0;
2947
2948         CHECK(n_tokens == 3, EINVAL);
2949
2950         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2951         CHECK(fdst, EINVAL);
2952         CHECK(!fdst->var_size, EINVAL);
2953
2954         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2955         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2956         if (fsrc) {
2957                 CHECK(!fsrc->var_size, EINVAL);
2958
2959                 instr->type = INSTR_ALU_SHR;
2960                 if (dst[0] == 'h' && src[0] != 'h')
2961                         instr->type = INSTR_ALU_SHR_HM;
2962                 if (dst[0] != 'h' && src[0] == 'h')
2963                         instr->type = INSTR_ALU_SHR_MH;
2964                 if (dst[0] == 'h' && src[0] == 'h')
2965                         instr->type = INSTR_ALU_SHR_HH;
2966
2967                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2968                 instr->alu.dst.n_bits = fdst->n_bits;
2969                 instr->alu.dst.offset = fdst->offset / 8;
2970                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2971                 instr->alu.src.n_bits = fsrc->n_bits;
2972                 instr->alu.src.offset = fsrc->offset / 8;
2973                 return 0;
2974         }
2975
2976         /* SHR_MI, SHR_HI. */
2977         src_val = strtoull(src, &src, 0);
2978         CHECK(!src[0], EINVAL);
2979
2980         instr->type = INSTR_ALU_SHR_MI;
2981         if (dst[0] == 'h')
2982                 instr->type = INSTR_ALU_SHR_HI;
2983
2984         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2985         instr->alu.dst.n_bits = fdst->n_bits;
2986         instr->alu.dst.offset = fdst->offset / 8;
2987         instr->alu.src_val = src_val;
2988         return 0;
2989 }
2990
2991 static int
2992 instr_alu_and_translate(struct rte_swx_pipeline *p,
2993                         struct action *action,
2994                         char **tokens,
2995                         int n_tokens,
2996                         struct instruction *instr,
2997                         struct instruction_data *data __rte_unused)
2998 {
2999         char *dst = tokens[1], *src = tokens[2];
3000         struct field *fdst, *fsrc;
3001         uint64_t src_val;
3002         uint32_t dst_struct_id = 0, src_struct_id = 0;
3003
3004         CHECK(n_tokens == 3, EINVAL);
3005
3006         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3007         CHECK(fdst, EINVAL);
3008         CHECK(!fdst->var_size, EINVAL);
3009
3010         /* AND, AND_MH, AND_HM, AND_HH. */
3011         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3012         if (fsrc) {
3013                 CHECK(!fsrc->var_size, EINVAL);
3014
3015                 instr->type = INSTR_ALU_AND;
3016                 if (dst[0] != 'h' && src[0] == 'h')
3017                         instr->type = INSTR_ALU_AND_MH;
3018                 if (dst[0] == 'h' && src[0] != 'h')
3019                         instr->type = INSTR_ALU_AND_HM;
3020                 if (dst[0] == 'h' && src[0] == 'h')
3021                         instr->type = INSTR_ALU_AND_HH;
3022
3023                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3024                 instr->alu.dst.n_bits = fdst->n_bits;
3025                 instr->alu.dst.offset = fdst->offset / 8;
3026                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3027                 instr->alu.src.n_bits = fsrc->n_bits;
3028                 instr->alu.src.offset = fsrc->offset / 8;
3029                 return 0;
3030         }
3031
3032         /* AND_I. */
3033         src_val = strtoull(src, &src, 0);
3034         CHECK(!src[0], EINVAL);
3035
3036         if (dst[0] == 'h')
3037                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3038
3039         instr->type = INSTR_ALU_AND_I;
3040         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3041         instr->alu.dst.n_bits = fdst->n_bits;
3042         instr->alu.dst.offset = fdst->offset / 8;
3043         instr->alu.src_val = src_val;
3044         return 0;
3045 }
3046
3047 static int
3048 instr_alu_or_translate(struct rte_swx_pipeline *p,
3049                        struct action *action,
3050                        char **tokens,
3051                        int n_tokens,
3052                        struct instruction *instr,
3053                        struct instruction_data *data __rte_unused)
3054 {
3055         char *dst = tokens[1], *src = tokens[2];
3056         struct field *fdst, *fsrc;
3057         uint64_t src_val;
3058         uint32_t dst_struct_id = 0, src_struct_id = 0;
3059
3060         CHECK(n_tokens == 3, EINVAL);
3061
3062         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3063         CHECK(fdst, EINVAL);
3064         CHECK(!fdst->var_size, EINVAL);
3065
3066         /* OR, OR_MH, OR_HM, OR_HH. */
3067         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3068         if (fsrc) {
3069                 CHECK(!fsrc->var_size, EINVAL);
3070
3071                 instr->type = INSTR_ALU_OR;
3072                 if (dst[0] != 'h' && src[0] == 'h')
3073                         instr->type = INSTR_ALU_OR_MH;
3074                 if (dst[0] == 'h' && src[0] != 'h')
3075                         instr->type = INSTR_ALU_OR_HM;
3076                 if (dst[0] == 'h' && src[0] == 'h')
3077                         instr->type = INSTR_ALU_OR_HH;
3078
3079                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3080                 instr->alu.dst.n_bits = fdst->n_bits;
3081                 instr->alu.dst.offset = fdst->offset / 8;
3082                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3083                 instr->alu.src.n_bits = fsrc->n_bits;
3084                 instr->alu.src.offset = fsrc->offset / 8;
3085                 return 0;
3086         }
3087
3088         /* OR_I. */
3089         src_val = strtoull(src, &src, 0);
3090         CHECK(!src[0], EINVAL);
3091
3092         if (dst[0] == 'h')
3093                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3094
3095         instr->type = INSTR_ALU_OR_I;
3096         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3097         instr->alu.dst.n_bits = fdst->n_bits;
3098         instr->alu.dst.offset = fdst->offset / 8;
3099         instr->alu.src_val = src_val;
3100         return 0;
3101 }
3102
3103 static int
3104 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3105                         struct action *action,
3106                         char **tokens,
3107                         int n_tokens,
3108                         struct instruction *instr,
3109                         struct instruction_data *data __rte_unused)
3110 {
3111         char *dst = tokens[1], *src = tokens[2];
3112         struct field *fdst, *fsrc;
3113         uint64_t src_val;
3114         uint32_t dst_struct_id = 0, src_struct_id = 0;
3115
3116         CHECK(n_tokens == 3, EINVAL);
3117
3118         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3119         CHECK(fdst, EINVAL);
3120         CHECK(!fdst->var_size, EINVAL);
3121
3122         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3123         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3124         if (fsrc) {
3125                 CHECK(!fsrc->var_size, EINVAL);
3126
3127                 instr->type = INSTR_ALU_XOR;
3128                 if (dst[0] != 'h' && src[0] == 'h')
3129                         instr->type = INSTR_ALU_XOR_MH;
3130                 if (dst[0] == 'h' && src[0] != 'h')
3131                         instr->type = INSTR_ALU_XOR_HM;
3132                 if (dst[0] == 'h' && src[0] == 'h')
3133                         instr->type = INSTR_ALU_XOR_HH;
3134
3135                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3136                 instr->alu.dst.n_bits = fdst->n_bits;
3137                 instr->alu.dst.offset = fdst->offset / 8;
3138                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3139                 instr->alu.src.n_bits = fsrc->n_bits;
3140                 instr->alu.src.offset = fsrc->offset / 8;
3141                 return 0;
3142         }
3143
3144         /* XOR_I. */
3145         src_val = strtoull(src, &src, 0);
3146         CHECK(!src[0], EINVAL);
3147
3148         if (dst[0] == 'h')
3149                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3150
3151         instr->type = INSTR_ALU_XOR_I;
3152         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3153         instr->alu.dst.n_bits = fdst->n_bits;
3154         instr->alu.dst.offset = fdst->offset / 8;
3155         instr->alu.src_val = src_val;
3156         return 0;
3157 }
3158
3159 static inline void
3160 instr_alu_add_exec(struct rte_swx_pipeline *p)
3161 {
3162         struct thread *t = &p->threads[p->thread_id];
3163         struct instruction *ip = t->ip;
3164
3165         /* Structs */
3166         __instr_alu_add_exec(p, t, ip);
3167
3168         /* Thread. */
3169         thread_ip_inc(p);
3170 }
3171
3172 static inline void
3173 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3174 {
3175         struct thread *t = &p->threads[p->thread_id];
3176         struct instruction *ip = t->ip;
3177
3178         /* Structs. */
3179         __instr_alu_add_mh_exec(p, t, ip);
3180
3181         /* Thread. */
3182         thread_ip_inc(p);
3183 }
3184
3185 static inline void
3186 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3187 {
3188         struct thread *t = &p->threads[p->thread_id];
3189         struct instruction *ip = t->ip;
3190
3191         /* Structs. */
3192         __instr_alu_add_hm_exec(p, t, ip);
3193
3194         /* Thread. */
3195         thread_ip_inc(p);
3196 }
3197
3198 static inline void
3199 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3200 {
3201         struct thread *t = &p->threads[p->thread_id];
3202         struct instruction *ip = t->ip;
3203
3204         /* Structs. */
3205         __instr_alu_add_hh_exec(p, t, ip);
3206
3207         /* Thread. */
3208         thread_ip_inc(p);
3209 }
3210
3211 static inline void
3212 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3213 {
3214         struct thread *t = &p->threads[p->thread_id];
3215         struct instruction *ip = t->ip;
3216
3217         /* Structs. */
3218         __instr_alu_add_mi_exec(p, t, ip);
3219
3220         /* Thread. */
3221         thread_ip_inc(p);
3222 }
3223
3224 static inline void
3225 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3226 {
3227         struct thread *t = &p->threads[p->thread_id];
3228         struct instruction *ip = t->ip;
3229
3230         /* Structs. */
3231         __instr_alu_add_hi_exec(p, t, ip);
3232
3233         /* Thread. */
3234         thread_ip_inc(p);
3235 }
3236
3237 static inline void
3238 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3239 {
3240         struct thread *t = &p->threads[p->thread_id];
3241         struct instruction *ip = t->ip;
3242
3243         /* Structs. */
3244         __instr_alu_sub_exec(p, t, ip);
3245
3246         /* Thread. */
3247         thread_ip_inc(p);
3248 }
3249
3250 static inline void
3251 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3252 {
3253         struct thread *t = &p->threads[p->thread_id];
3254         struct instruction *ip = t->ip;
3255
3256         /* Structs. */
3257         __instr_alu_sub_mh_exec(p, t, ip);
3258
3259         /* Thread. */
3260         thread_ip_inc(p);
3261 }
3262
3263 static inline void
3264 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3265 {
3266         struct thread *t = &p->threads[p->thread_id];
3267         struct instruction *ip = t->ip;
3268
3269         /* Structs. */
3270         __instr_alu_sub_hm_exec(p, t, ip);
3271
3272         /* Thread. */
3273         thread_ip_inc(p);
3274 }
3275
3276 static inline void
3277 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3278 {
3279         struct thread *t = &p->threads[p->thread_id];
3280         struct instruction *ip = t->ip;
3281
3282         /* Structs. */
3283         __instr_alu_sub_hh_exec(p, t, ip);
3284
3285         /* Thread. */
3286         thread_ip_inc(p);
3287 }
3288
3289 static inline void
3290 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3291 {
3292         struct thread *t = &p->threads[p->thread_id];
3293         struct instruction *ip = t->ip;
3294
3295         /* Structs. */
3296         __instr_alu_sub_mi_exec(p, t, ip);
3297
3298         /* Thread. */
3299         thread_ip_inc(p);
3300 }
3301
3302 static inline void
3303 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3304 {
3305         struct thread *t = &p->threads[p->thread_id];
3306         struct instruction *ip = t->ip;
3307
3308         /* Structs. */
3309         __instr_alu_sub_hi_exec(p, t, ip);
3310
3311         /* Thread. */
3312         thread_ip_inc(p);
3313 }
3314
3315 static inline void
3316 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3317 {
3318         struct thread *t = &p->threads[p->thread_id];
3319         struct instruction *ip = t->ip;
3320
3321         /* Structs. */
3322         __instr_alu_shl_exec(p, t, ip);
3323
3324         /* Thread. */
3325         thread_ip_inc(p);
3326 }
3327
3328 static inline void
3329 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3330 {
3331         struct thread *t = &p->threads[p->thread_id];
3332         struct instruction *ip = t->ip;
3333
3334         /* Structs. */
3335         __instr_alu_shl_mh_exec(p, t, ip);
3336
3337         /* Thread. */
3338         thread_ip_inc(p);
3339 }
3340
3341 static inline void
3342 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3343 {
3344         struct thread *t = &p->threads[p->thread_id];
3345         struct instruction *ip = t->ip;
3346
3347         /* Structs. */
3348         __instr_alu_shl_hm_exec(p, t, ip);
3349
3350         /* Thread. */
3351         thread_ip_inc(p);
3352 }
3353
3354 static inline void
3355 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3356 {
3357         struct thread *t = &p->threads[p->thread_id];
3358         struct instruction *ip = t->ip;
3359
3360         /* Structs. */
3361         __instr_alu_shl_hh_exec(p, t, ip);
3362
3363         /* Thread. */
3364         thread_ip_inc(p);
3365 }
3366
3367 static inline void
3368 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3369 {
3370         struct thread *t = &p->threads[p->thread_id];
3371         struct instruction *ip = t->ip;
3372
3373         /* Structs. */
3374         __instr_alu_shl_mi_exec(p, t, ip);
3375
3376         /* Thread. */
3377         thread_ip_inc(p);
3378 }
3379
3380 static inline void
3381 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3382 {
3383         struct thread *t = &p->threads[p->thread_id];
3384         struct instruction *ip = t->ip;
3385
3386         /* Structs. */
3387         __instr_alu_shl_hi_exec(p, t, ip);
3388
3389         /* Thread. */
3390         thread_ip_inc(p);
3391 }
3392
3393 static inline void
3394 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3395 {
3396         struct thread *t = &p->threads[p->thread_id];
3397         struct instruction *ip = t->ip;
3398
3399         /* Structs. */
3400         __instr_alu_shr_exec(p, t, ip);
3401
3402         /* Thread. */
3403         thread_ip_inc(p);
3404 }
3405
3406 static inline void
3407 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3408 {
3409         struct thread *t = &p->threads[p->thread_id];
3410         struct instruction *ip = t->ip;
3411
3412         /* Structs. */
3413         __instr_alu_shr_mh_exec(p, t, ip);
3414
3415         /* Thread. */
3416         thread_ip_inc(p);
3417 }
3418
3419 static inline void
3420 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3421 {
3422         struct thread *t = &p->threads[p->thread_id];
3423         struct instruction *ip = t->ip;
3424
3425         /* Structs. */
3426         __instr_alu_shr_hm_exec(p, t, ip);
3427
3428         /* Thread. */
3429         thread_ip_inc(p);
3430 }
3431
3432 static inline void
3433 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3434 {
3435         struct thread *t = &p->threads[p->thread_id];
3436         struct instruction *ip = t->ip;
3437
3438         /* Structs. */
3439         __instr_alu_shr_hh_exec(p, t, ip);
3440
3441         /* Thread. */
3442         thread_ip_inc(p);
3443 }
3444
3445 static inline void
3446 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3447 {
3448         struct thread *t = &p->threads[p->thread_id];
3449         struct instruction *ip = t->ip;
3450
3451         /* Structs. */
3452         __instr_alu_shr_mi_exec(p, t, ip);
3453
3454         /* Thread. */
3455         thread_ip_inc(p);
3456 }
3457
3458 static inline void
3459 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3460 {
3461         struct thread *t = &p->threads[p->thread_id];
3462         struct instruction *ip = t->ip;
3463
3464         /* Structs. */
3465         __instr_alu_shr_hi_exec(p, t, ip);
3466
3467         /* Thread. */
3468         thread_ip_inc(p);
3469 }
3470
3471 static inline void
3472 instr_alu_and_exec(struct rte_swx_pipeline *p)
3473 {
3474         struct thread *t = &p->threads[p->thread_id];
3475         struct instruction *ip = t->ip;
3476
3477         /* Structs. */
3478         __instr_alu_and_exec(p, t, ip);
3479
3480         /* Thread. */
3481         thread_ip_inc(p);
3482 }
3483
3484 static inline void
3485 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3486 {
3487         struct thread *t = &p->threads[p->thread_id];
3488         struct instruction *ip = t->ip;
3489
3490         /* Structs. */
3491         __instr_alu_and_mh_exec(p, t, ip);
3492
3493         /* Thread. */
3494         thread_ip_inc(p);
3495 }
3496
3497 static inline void
3498 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3499 {
3500         struct thread *t = &p->threads[p->thread_id];
3501         struct instruction *ip = t->ip;
3502
3503         /* Structs. */
3504         __instr_alu_and_hm_exec(p, t, ip);
3505
3506         /* Thread. */
3507         thread_ip_inc(p);
3508 }
3509
3510 static inline void
3511 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3512 {
3513         struct thread *t = &p->threads[p->thread_id];
3514         struct instruction *ip = t->ip;
3515
3516         /* Structs. */
3517         __instr_alu_and_hh_exec(p, t, ip);
3518
3519         /* Thread. */
3520         thread_ip_inc(p);
3521 }
3522
3523 static inline void
3524 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3525 {
3526         struct thread *t = &p->threads[p->thread_id];
3527         struct instruction *ip = t->ip;
3528
3529         /* Structs. */
3530         __instr_alu_and_i_exec(p, t, ip);
3531
3532         /* Thread. */
3533         thread_ip_inc(p);
3534 }
3535
3536 static inline void
3537 instr_alu_or_exec(struct rte_swx_pipeline *p)
3538 {
3539         struct thread *t = &p->threads[p->thread_id];
3540         struct instruction *ip = t->ip;
3541
3542         /* Structs. */
3543         __instr_alu_or_exec(p, t, ip);
3544
3545         /* Thread. */
3546         thread_ip_inc(p);
3547 }
3548
3549 static inline void
3550 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3551 {
3552         struct thread *t = &p->threads[p->thread_id];
3553         struct instruction *ip = t->ip;
3554
3555         /* Structs. */
3556         __instr_alu_or_mh_exec(p, t, ip);
3557
3558         /* Thread. */
3559         thread_ip_inc(p);
3560 }
3561
3562 static inline void
3563 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3564 {
3565         struct thread *t = &p->threads[p->thread_id];
3566         struct instruction *ip = t->ip;
3567
3568         /* Structs. */
3569         __instr_alu_or_hm_exec(p, t, ip);
3570
3571         /* Thread. */
3572         thread_ip_inc(p);
3573 }
3574
3575 static inline void
3576 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3577 {
3578         struct thread *t = &p->threads[p->thread_id];
3579         struct instruction *ip = t->ip;
3580
3581         /* Structs. */
3582         __instr_alu_or_hh_exec(p, t, ip);
3583
3584         /* Thread. */
3585         thread_ip_inc(p);
3586 }
3587
3588 static inline void
3589 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3590 {
3591         struct thread *t = &p->threads[p->thread_id];
3592         struct instruction *ip = t->ip;
3593
3594         /* Structs. */
3595         __instr_alu_or_i_exec(p, t, ip);
3596
3597         /* Thread. */
3598         thread_ip_inc(p);
3599 }
3600
3601 static inline void
3602 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3603 {
3604         struct thread *t = &p->threads[p->thread_id];
3605         struct instruction *ip = t->ip;
3606
3607         /* Structs. */
3608         __instr_alu_xor_exec(p, t, ip);
3609
3610         /* Thread. */
3611         thread_ip_inc(p);
3612 }
3613
3614 static inline void
3615 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3616 {
3617         struct thread *t = &p->threads[p->thread_id];
3618         struct instruction *ip = t->ip;
3619
3620         /* Structs. */
3621         __instr_alu_xor_mh_exec(p, t, ip);
3622
3623         /* Thread. */
3624         thread_ip_inc(p);
3625 }
3626
3627 static inline void
3628 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3629 {
3630         struct thread *t = &p->threads[p->thread_id];
3631         struct instruction *ip = t->ip;
3632
3633         /* Structs. */
3634         __instr_alu_xor_hm_exec(p, t, ip);
3635
3636         /* Thread. */
3637         thread_ip_inc(p);
3638 }
3639
3640 static inline void
3641 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3642 {
3643         struct thread *t = &p->threads[p->thread_id];
3644         struct instruction *ip = t->ip;
3645
3646         /* Structs. */
3647         __instr_alu_xor_hh_exec(p, t, ip);
3648
3649         /* Thread. */
3650         thread_ip_inc(p);
3651 }
3652
3653 static inline void
3654 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3655 {
3656         struct thread *t = &p->threads[p->thread_id];
3657         struct instruction *ip = t->ip;
3658
3659         /* Structs. */
3660         __instr_alu_xor_i_exec(p, t, ip);
3661
3662         /* Thread. */
3663         thread_ip_inc(p);
3664 }
3665
3666 static inline void
3667 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3668 {
3669         struct thread *t = &p->threads[p->thread_id];
3670         struct instruction *ip = t->ip;
3671
3672         /* Structs. */
3673         __instr_alu_ckadd_field_exec(p, t, ip);
3674
3675         /* Thread. */
3676         thread_ip_inc(p);
3677 }
3678
3679 static inline void
3680 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3681 {
3682         struct thread *t = &p->threads[p->thread_id];
3683         struct instruction *ip = t->ip;
3684
3685         /* Structs. */
3686         __instr_alu_cksub_field_exec(p, t, ip);
3687
3688         /* Thread. */
3689         thread_ip_inc(p);
3690 }
3691
3692 static inline void
3693 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3694 {
3695         struct thread *t = &p->threads[p->thread_id];
3696         struct instruction *ip = t->ip;
3697
3698         /* Structs. */
3699         __instr_alu_ckadd_struct20_exec(p, t, ip);
3700
3701         /* Thread. */
3702         thread_ip_inc(p);
3703 }
3704
3705 static inline void
3706 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3707 {
3708         struct thread *t = &p->threads[p->thread_id];
3709         struct instruction *ip = t->ip;
3710
3711         /* Structs. */
3712         __instr_alu_ckadd_struct_exec(p, t, ip);
3713
3714         /* Thread. */
3715         thread_ip_inc(p);
3716 }
3717
3718 /*
3719  * Register array.
3720  */
3721 static struct regarray *
3722 regarray_find(struct rte_swx_pipeline *p, const char *name);
3723
3724 static int
3725 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3726                       struct action *action,
3727                       char **tokens,
3728                       int n_tokens,
3729                       struct instruction *instr,
3730                       struct instruction_data *data __rte_unused)
3731 {
3732         char *regarray = tokens[1], *idx = tokens[2];
3733         struct regarray *r;
3734         struct field *fidx;
3735         uint32_t idx_struct_id, idx_val;
3736
3737         CHECK(n_tokens == 3, EINVAL);
3738
3739         r = regarray_find(p, regarray);
3740         CHECK(r, EINVAL);
3741
3742         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3743         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3744         if (fidx) {
3745                 CHECK(!fidx->var_size, EINVAL);
3746
3747                 instr->type = INSTR_REGPREFETCH_RM;
3748                 if (idx[0] == 'h')
3749                         instr->type = INSTR_REGPREFETCH_RH;
3750
3751                 instr->regarray.regarray_id = r->id;
3752                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3753                 instr->regarray.idx.n_bits = fidx->n_bits;
3754                 instr->regarray.idx.offset = fidx->offset / 8;
3755                 instr->regarray.dstsrc_val = 0; /* Unused. */
3756                 return 0;
3757         }
3758
3759         /* REGPREFETCH_RI. */
3760         idx_val = strtoul(idx, &idx, 0);
3761         CHECK(!idx[0], EINVAL);
3762
3763         instr->type = INSTR_REGPREFETCH_RI;
3764         instr->regarray.regarray_id = r->id;
3765         instr->regarray.idx_val = idx_val;
3766         instr->regarray.dstsrc_val = 0; /* Unused. */
3767         return 0;
3768 }
3769
3770 static int
3771 instr_regrd_translate(struct rte_swx_pipeline *p,
3772                       struct action *action,
3773                       char **tokens,
3774                       int n_tokens,
3775                       struct instruction *instr,
3776                       struct instruction_data *data __rte_unused)
3777 {
3778         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3779         struct regarray *r;
3780         struct field *fdst, *fidx;
3781         uint32_t dst_struct_id, idx_struct_id, idx_val;
3782
3783         CHECK(n_tokens == 4, EINVAL);
3784
3785         r = regarray_find(p, regarray);
3786         CHECK(r, EINVAL);
3787
3788         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3789         CHECK(fdst, EINVAL);
3790         CHECK(!fdst->var_size, EINVAL);
3791
3792         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3793         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3794         if (fidx) {
3795                 CHECK(!fidx->var_size, EINVAL);
3796
3797                 instr->type = INSTR_REGRD_MRM;
3798                 if (dst[0] == 'h' && idx[0] != 'h')
3799                         instr->type = INSTR_REGRD_HRM;
3800                 if (dst[0] != 'h' && idx[0] == 'h')
3801                         instr->type = INSTR_REGRD_MRH;
3802                 if (dst[0] == 'h' && idx[0] == 'h')
3803                         instr->type = INSTR_REGRD_HRH;
3804
3805                 instr->regarray.regarray_id = r->id;
3806                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3807                 instr->regarray.idx.n_bits = fidx->n_bits;
3808                 instr->regarray.idx.offset = fidx->offset / 8;
3809                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3810                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3811                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3812                 return 0;
3813         }
3814
3815         /* REGRD_MRI, REGRD_HRI. */
3816         idx_val = strtoul(idx, &idx, 0);
3817         CHECK(!idx[0], EINVAL);
3818
3819         instr->type = INSTR_REGRD_MRI;
3820         if (dst[0] == 'h')
3821                 instr->type = INSTR_REGRD_HRI;
3822
3823         instr->regarray.regarray_id = r->id;
3824         instr->regarray.idx_val = idx_val;
3825         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3826         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3827         instr->regarray.dstsrc.offset = fdst->offset / 8;
3828         return 0;
3829 }
3830
3831 static int
3832 instr_regwr_translate(struct rte_swx_pipeline *p,
3833                       struct action *action,
3834                       char **tokens,
3835                       int n_tokens,
3836                       struct instruction *instr,
3837                       struct instruction_data *data __rte_unused)
3838 {
3839         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3840         struct regarray *r;
3841         struct field *fidx, *fsrc;
3842         uint64_t src_val;
3843         uint32_t idx_struct_id, idx_val, src_struct_id;
3844
3845         CHECK(n_tokens == 4, EINVAL);
3846
3847         r = regarray_find(p, regarray);
3848         CHECK(r, EINVAL);
3849
3850         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3851         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3852         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3853         if (fidx && fsrc) {
3854                 CHECK(!fidx->var_size, EINVAL);
3855                 CHECK(!fsrc->var_size, EINVAL);
3856
3857                 instr->type = INSTR_REGWR_RMM;
3858                 if (idx[0] == 'h' && src[0] != 'h')
3859                         instr->type = INSTR_REGWR_RHM;
3860                 if (idx[0] != 'h' && src[0] == 'h')
3861                         instr->type = INSTR_REGWR_RMH;
3862                 if (idx[0] == 'h' && src[0] == 'h')
3863                         instr->type = INSTR_REGWR_RHH;
3864
3865                 instr->regarray.regarray_id = r->id;
3866                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3867                 instr->regarray.idx.n_bits = fidx->n_bits;
3868                 instr->regarray.idx.offset = fidx->offset / 8;
3869                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3870                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3871                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3872                 return 0;
3873         }
3874
3875         /* REGWR_RHI, REGWR_RMI. */
3876         if (fidx && !fsrc) {
3877                 CHECK(!fidx->var_size, EINVAL);
3878
3879                 src_val = strtoull(src, &src, 0);
3880                 CHECK(!src[0], EINVAL);
3881
3882                 instr->type = INSTR_REGWR_RMI;
3883                 if (idx[0] == 'h')
3884                         instr->type = INSTR_REGWR_RHI;
3885
3886                 instr->regarray.regarray_id = r->id;
3887                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3888                 instr->regarray.idx.n_bits = fidx->n_bits;
3889                 instr->regarray.idx.offset = fidx->offset / 8;
3890                 instr->regarray.dstsrc_val = src_val;
3891                 return 0;
3892         }
3893
3894         /* REGWR_RIH, REGWR_RIM. */
3895         if (!fidx && fsrc) {
3896                 idx_val = strtoul(idx, &idx, 0);
3897                 CHECK(!idx[0], EINVAL);
3898
3899                 CHECK(!fsrc->var_size, EINVAL);
3900
3901                 instr->type = INSTR_REGWR_RIM;
3902                 if (src[0] == 'h')
3903                         instr->type = INSTR_REGWR_RIH;
3904
3905                 instr->regarray.regarray_id = r->id;
3906                 instr->regarray.idx_val = idx_val;
3907                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3908                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3909                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3910                 return 0;
3911         }
3912
3913         /* REGWR_RII. */
3914         src_val = strtoull(src, &src, 0);
3915         CHECK(!src[0], EINVAL);
3916
3917         idx_val = strtoul(idx, &idx, 0);
3918         CHECK(!idx[0], EINVAL);
3919
3920         instr->type = INSTR_REGWR_RII;
3921         instr->regarray.idx_val = idx_val;
3922         instr->regarray.dstsrc_val = src_val;
3923
3924         return 0;
3925 }
3926
3927 static int
3928 instr_regadd_translate(struct rte_swx_pipeline *p,
3929                        struct action *action,
3930                        char **tokens,
3931                        int n_tokens,
3932                        struct instruction *instr,
3933                        struct instruction_data *data __rte_unused)
3934 {
3935         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3936         struct regarray *r;
3937         struct field *fidx, *fsrc;
3938         uint64_t src_val;
3939         uint32_t idx_struct_id, idx_val, src_struct_id;
3940
3941         CHECK(n_tokens == 4, EINVAL);
3942
3943         r = regarray_find(p, regarray);
3944         CHECK(r, EINVAL);
3945
3946         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3947         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3948         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3949         if (fidx && fsrc) {
3950                 CHECK(!fidx->var_size, EINVAL);
3951                 CHECK(!fsrc->var_size, EINVAL);
3952
3953                 instr->type = INSTR_REGADD_RMM;
3954                 if (idx[0] == 'h' && src[0] != 'h')
3955                         instr->type = INSTR_REGADD_RHM;
3956                 if (idx[0] != 'h' && src[0] == 'h')
3957                         instr->type = INSTR_REGADD_RMH;
3958                 if (idx[0] == 'h' && src[0] == 'h')
3959                         instr->type = INSTR_REGADD_RHH;
3960
3961                 instr->regarray.regarray_id = r->id;
3962                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3963                 instr->regarray.idx.n_bits = fidx->n_bits;
3964                 instr->regarray.idx.offset = fidx->offset / 8;
3965                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3966                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3967                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3968                 return 0;
3969         }
3970
3971         /* REGADD_RHI, REGADD_RMI. */
3972         if (fidx && !fsrc) {
3973                 CHECK(!fidx->var_size, EINVAL);
3974
3975                 src_val = strtoull(src, &src, 0);
3976                 CHECK(!src[0], EINVAL);
3977
3978                 instr->type = INSTR_REGADD_RMI;
3979                 if (idx[0] == 'h')
3980                         instr->type = INSTR_REGADD_RHI;
3981
3982                 instr->regarray.regarray_id = r->id;
3983                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3984                 instr->regarray.idx.n_bits = fidx->n_bits;
3985                 instr->regarray.idx.offset = fidx->offset / 8;
3986                 instr->regarray.dstsrc_val = src_val;
3987                 return 0;
3988         }
3989
3990         /* REGADD_RIH, REGADD_RIM. */
3991         if (!fidx && fsrc) {
3992                 idx_val = strtoul(idx, &idx, 0);
3993                 CHECK(!idx[0], EINVAL);
3994
3995                 CHECK(!fsrc->var_size, EINVAL);
3996
3997                 instr->type = INSTR_REGADD_RIM;
3998                 if (src[0] == 'h')
3999                         instr->type = INSTR_REGADD_RIH;
4000
4001                 instr->regarray.regarray_id = r->id;
4002                 instr->regarray.idx_val = idx_val;
4003                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4004                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4005                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4006                 return 0;
4007         }
4008
4009         /* REGADD_RII. */
4010         src_val = strtoull(src, &src, 0);
4011         CHECK(!src[0], EINVAL);
4012
4013         idx_val = strtoul(idx, &idx, 0);
4014         CHECK(!idx[0], EINVAL);
4015
4016         instr->type = INSTR_REGADD_RII;
4017         instr->regarray.idx_val = idx_val;
4018         instr->regarray.dstsrc_val = src_val;
4019         return 0;
4020 }
4021
4022 static inline void
4023 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4024 {
4025         struct thread *t = &p->threads[p->thread_id];
4026         struct instruction *ip = t->ip;
4027
4028         /* Structs. */
4029         __instr_regprefetch_rh_exec(p, t, ip);
4030
4031         /* Thread. */
4032         thread_ip_inc(p);
4033 }
4034
4035 static inline void
4036 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4037 {
4038         struct thread *t = &p->threads[p->thread_id];
4039         struct instruction *ip = t->ip;
4040
4041         /* Structs. */
4042         __instr_regprefetch_rm_exec(p, t, ip);
4043
4044         /* Thread. */
4045         thread_ip_inc(p);
4046 }
4047
4048 static inline void
4049 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4050 {
4051         struct thread *t = &p->threads[p->thread_id];
4052         struct instruction *ip = t->ip;
4053
4054         /* Structs. */
4055         __instr_regprefetch_ri_exec(p, t, ip);
4056
4057         /* Thread. */
4058         thread_ip_inc(p);
4059 }
4060
4061 static inline void
4062 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4063 {
4064         struct thread *t = &p->threads[p->thread_id];
4065         struct instruction *ip = t->ip;
4066
4067         /* Structs. */
4068         __instr_regrd_hrh_exec(p, t, ip);
4069
4070         /* Thread. */
4071         thread_ip_inc(p);
4072 }
4073
4074 static inline void
4075 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4076 {
4077         struct thread *t = &p->threads[p->thread_id];
4078         struct instruction *ip = t->ip;
4079
4080         /* Structs. */
4081         __instr_regrd_hrm_exec(p, t, ip);
4082
4083         /* Thread. */
4084         thread_ip_inc(p);
4085 }
4086
4087 static inline void
4088 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4089 {
4090         struct thread *t = &p->threads[p->thread_id];
4091         struct instruction *ip = t->ip;
4092
4093         /* Structs. */
4094         __instr_regrd_mrh_exec(p, t, ip);
4095
4096         /* Thread. */
4097         thread_ip_inc(p);
4098 }
4099
4100 static inline void
4101 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4102 {
4103         struct thread *t = &p->threads[p->thread_id];
4104         struct instruction *ip = t->ip;
4105
4106         /* Structs. */
4107         __instr_regrd_mrm_exec(p, t, ip);
4108
4109         /* Thread. */
4110         thread_ip_inc(p);
4111 }
4112
4113 static inline void
4114 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4115 {
4116         struct thread *t = &p->threads[p->thread_id];
4117         struct instruction *ip = t->ip;
4118
4119         /* Structs. */
4120         __instr_regrd_hri_exec(p, t, ip);
4121
4122         /* Thread. */
4123         thread_ip_inc(p);
4124 }
4125
4126 static inline void
4127 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4128 {
4129         struct thread *t = &p->threads[p->thread_id];
4130         struct instruction *ip = t->ip;
4131
4132         /* Structs. */
4133         __instr_regrd_mri_exec(p, t, ip);
4134
4135         /* Thread. */
4136         thread_ip_inc(p);
4137 }
4138
4139 static inline void
4140 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4141 {
4142         struct thread *t = &p->threads[p->thread_id];
4143         struct instruction *ip = t->ip;
4144
4145         /* Structs. */
4146         __instr_regwr_rhh_exec(p, t, ip);
4147
4148         /* Thread. */
4149         thread_ip_inc(p);
4150 }
4151
4152 static inline void
4153 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4154 {
4155         struct thread *t = &p->threads[p->thread_id];
4156         struct instruction *ip = t->ip;
4157
4158         /* Structs. */
4159         __instr_regwr_rhm_exec(p, t, ip);
4160
4161         /* Thread. */
4162         thread_ip_inc(p);
4163 }
4164
4165 static inline void
4166 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4167 {
4168         struct thread *t = &p->threads[p->thread_id];
4169         struct instruction *ip = t->ip;
4170
4171         /* Structs. */
4172         __instr_regwr_rmh_exec(p, t, ip);
4173
4174         /* Thread. */
4175         thread_ip_inc(p);
4176 }
4177
4178 static inline void
4179 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4180 {
4181         struct thread *t = &p->threads[p->thread_id];
4182         struct instruction *ip = t->ip;
4183
4184         /* Structs. */
4185         __instr_regwr_rmm_exec(p, t, ip);
4186
4187         /* Thread. */
4188         thread_ip_inc(p);
4189 }
4190
4191 static inline void
4192 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4193 {
4194         struct thread *t = &p->threads[p->thread_id];
4195         struct instruction *ip = t->ip;
4196
4197         /* Structs. */
4198         __instr_regwr_rhi_exec(p, t, ip);
4199
4200         /* Thread. */
4201         thread_ip_inc(p);
4202 }
4203
4204 static inline void
4205 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4206 {
4207         struct thread *t = &p->threads[p->thread_id];
4208         struct instruction *ip = t->ip;
4209
4210         /* Structs. */
4211         __instr_regwr_rmi_exec(p, t, ip);
4212
4213         /* Thread. */
4214         thread_ip_inc(p);
4215 }
4216
4217 static inline void
4218 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4219 {
4220         struct thread *t = &p->threads[p->thread_id];
4221         struct instruction *ip = t->ip;
4222
4223         /* Structs. */
4224         __instr_regwr_rih_exec(p, t, ip);
4225
4226         /* Thread. */
4227         thread_ip_inc(p);
4228 }
4229
4230 static inline void
4231 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4232 {
4233         struct thread *t = &p->threads[p->thread_id];
4234         struct instruction *ip = t->ip;
4235
4236         /* Structs. */
4237         __instr_regwr_rim_exec(p, t, ip);
4238
4239         /* Thread. */
4240         thread_ip_inc(p);
4241 }
4242
4243 static inline void
4244 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4245 {
4246         struct thread *t = &p->threads[p->thread_id];
4247         struct instruction *ip = t->ip;
4248
4249         /* Structs. */
4250         __instr_regwr_rii_exec(p, t, ip);
4251
4252         /* Thread. */
4253         thread_ip_inc(p);
4254 }
4255
4256 static inline void
4257 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4258 {
4259         struct thread *t = &p->threads[p->thread_id];
4260         struct instruction *ip = t->ip;
4261
4262         /* Structs. */
4263         __instr_regadd_rhh_exec(p, t, ip);
4264
4265         /* Thread. */
4266         thread_ip_inc(p);
4267 }
4268
4269 static inline void
4270 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4271 {
4272         struct thread *t = &p->threads[p->thread_id];
4273         struct instruction *ip = t->ip;
4274
4275         /* Structs. */
4276         __instr_regadd_rhm_exec(p, t, ip);
4277
4278         /* Thread. */
4279         thread_ip_inc(p);
4280 }
4281
4282 static inline void
4283 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4284 {
4285         struct thread *t = &p->threads[p->thread_id];
4286         struct instruction *ip = t->ip;
4287
4288         /* Structs. */
4289         __instr_regadd_rmh_exec(p, t, ip);
4290
4291         /* Thread. */
4292         thread_ip_inc(p);
4293 }
4294
4295 static inline void
4296 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4297 {
4298         struct thread *t = &p->threads[p->thread_id];
4299         struct instruction *ip = t->ip;
4300
4301         /* Structs. */
4302         __instr_regadd_rmm_exec(p, t, ip);
4303
4304         /* Thread. */
4305         thread_ip_inc(p);
4306 }
4307
4308 static inline void
4309 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4310 {
4311         struct thread *t = &p->threads[p->thread_id];
4312         struct instruction *ip = t->ip;
4313
4314         /* Structs. */
4315         __instr_regadd_rhi_exec(p, t, ip);
4316
4317         /* Thread. */
4318         thread_ip_inc(p);
4319 }
4320
4321 static inline void
4322 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4323 {
4324         struct thread *t = &p->threads[p->thread_id];
4325         struct instruction *ip = t->ip;
4326
4327         /* Structs. */
4328         __instr_regadd_rmi_exec(p, t, ip);
4329
4330         /* Thread. */
4331         thread_ip_inc(p);
4332 }
4333
4334 static inline void
4335 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4336 {
4337         struct thread *t = &p->threads[p->thread_id];
4338         struct instruction *ip = t->ip;
4339
4340         /* Structs. */
4341         __instr_regadd_rih_exec(p, t, ip);
4342
4343         /* Thread. */
4344         thread_ip_inc(p);
4345 }
4346
4347 static inline void
4348 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4349 {
4350         struct thread *t = &p->threads[p->thread_id];
4351         struct instruction *ip = t->ip;
4352
4353         /* Structs. */
4354         __instr_regadd_rim_exec(p, t, ip);
4355
4356         /* Thread. */
4357         thread_ip_inc(p);
4358 }
4359
4360 static inline void
4361 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4362 {
4363         struct thread *t = &p->threads[p->thread_id];
4364         struct instruction *ip = t->ip;
4365
4366         /* Structs. */
4367         __instr_regadd_rii_exec(p, t, ip);
4368
4369         /* Thread. */
4370         thread_ip_inc(p);
4371 }
4372
4373 /*
4374  * metarray.
4375  */
4376 static struct metarray *
4377 metarray_find(struct rte_swx_pipeline *p, const char *name);
4378
4379 static int
4380 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4381                             struct action *action,
4382                             char **tokens,
4383                             int n_tokens,
4384                             struct instruction *instr,
4385                             struct instruction_data *data __rte_unused)
4386 {
4387         char *metarray = tokens[1], *idx = tokens[2];
4388         struct metarray *m;
4389         struct field *fidx;
4390         uint32_t idx_struct_id, idx_val;
4391
4392         CHECK(n_tokens == 3, EINVAL);
4393
4394         m = metarray_find(p, metarray);
4395         CHECK(m, EINVAL);
4396
4397         /* METPREFETCH_H, METPREFETCH_M. */
4398         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4399         if (fidx) {
4400                 CHECK(!fidx->var_size, EINVAL);
4401
4402                 instr->type = INSTR_METPREFETCH_M;
4403                 if (idx[0] == 'h')
4404                         instr->type = INSTR_METPREFETCH_H;
4405
4406                 instr->meter.metarray_id = m->id;
4407                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4408                 instr->meter.idx.n_bits = fidx->n_bits;
4409                 instr->meter.idx.offset = fidx->offset / 8;
4410                 return 0;
4411         }
4412
4413         /* METPREFETCH_I. */
4414         idx_val = strtoul(idx, &idx, 0);
4415         CHECK(!idx[0], EINVAL);
4416
4417         instr->type = INSTR_METPREFETCH_I;
4418         instr->meter.metarray_id = m->id;
4419         instr->meter.idx_val = idx_val;
4420         return 0;
4421 }
4422
4423 static int
4424 instr_meter_translate(struct rte_swx_pipeline *p,
4425                       struct action *action,
4426                       char **tokens,
4427                       int n_tokens,
4428                       struct instruction *instr,
4429                       struct instruction_data *data __rte_unused)
4430 {
4431         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4432         char *color_in = tokens[4], *color_out = tokens[5];
4433         struct metarray *m;
4434         struct field *fidx, *flength, *fcin, *fcout;
4435         uint32_t idx_struct_id, length_struct_id;
4436         uint32_t color_in_struct_id, color_out_struct_id;
4437
4438         CHECK(n_tokens == 6, EINVAL);
4439
4440         m = metarray_find(p, metarray);
4441         CHECK(m, EINVAL);
4442
4443         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4444
4445         flength = struct_field_parse(p, action, length, &length_struct_id);
4446         CHECK(flength, EINVAL);
4447         CHECK(!flength->var_size, EINVAL);
4448
4449         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4450
4451         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4452         CHECK(fcout, EINVAL);
4453         CHECK(!fcout->var_size, EINVAL);
4454
4455         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4456         if (fidx && fcin) {
4457                 CHECK(!fidx->var_size, EINVAL);
4458                 CHECK(!fcin->var_size, EINVAL);
4459
4460                 instr->type = INSTR_METER_MMM;
4461                 if (idx[0] == 'h' && length[0] == 'h')
4462                         instr->type = INSTR_METER_HHM;
4463                 if (idx[0] == 'h' && length[0] != 'h')
4464                         instr->type = INSTR_METER_HMM;
4465                 if (idx[0] != 'h' && length[0] == 'h')
4466                         instr->type = INSTR_METER_MHM;
4467
4468                 instr->meter.metarray_id = m->id;
4469
4470                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4471                 instr->meter.idx.n_bits = fidx->n_bits;
4472                 instr->meter.idx.offset = fidx->offset / 8;
4473
4474                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4475                 instr->meter.length.n_bits = flength->n_bits;
4476                 instr->meter.length.offset = flength->offset / 8;
4477
4478                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4479                 instr->meter.color_in.n_bits = fcin->n_bits;
4480                 instr->meter.color_in.offset = fcin->offset / 8;
4481
4482                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4483                 instr->meter.color_out.n_bits = fcout->n_bits;
4484                 instr->meter.color_out.offset = fcout->offset / 8;
4485
4486                 return 0;
4487         }
4488
4489         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4490         if (fidx && !fcin) {
4491                 uint32_t color_in_val;
4492
4493                 CHECK(!fidx->var_size, EINVAL);
4494
4495                 color_in_val = strtoul(color_in, &color_in, 0);
4496                 CHECK(!color_in[0], EINVAL);
4497
4498                 instr->type = INSTR_METER_MMI;
4499                 if (idx[0] == 'h' && length[0] == 'h')
4500                         instr->type = INSTR_METER_HHI;
4501                 if (idx[0] == 'h' && length[0] != 'h')
4502                         instr->type = INSTR_METER_HMI;
4503                 if (idx[0] != 'h' && length[0] == 'h')
4504                         instr->type = INSTR_METER_MHI;
4505
4506                 instr->meter.metarray_id = m->id;
4507
4508                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4509                 instr->meter.idx.n_bits = fidx->n_bits;
4510                 instr->meter.idx.offset = fidx->offset / 8;
4511
4512                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4513                 instr->meter.length.n_bits = flength->n_bits;
4514                 instr->meter.length.offset = flength->offset / 8;
4515
4516                 instr->meter.color_in_val = color_in_val;
4517
4518                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4519                 instr->meter.color_out.n_bits = fcout->n_bits;
4520                 instr->meter.color_out.offset = fcout->offset / 8;
4521
4522                 return 0;
4523         }
4524
4525         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4526         if (!fidx && fcin) {
4527                 uint32_t idx_val;
4528
4529                 idx_val = strtoul(idx, &idx, 0);
4530                 CHECK(!idx[0], EINVAL);
4531
4532                 CHECK(!fcin->var_size, EINVAL);
4533
4534                 instr->type = INSTR_METER_IMM;
4535                 if (length[0] == 'h')
4536                         instr->type = INSTR_METER_IHM;
4537
4538                 instr->meter.metarray_id = m->id;
4539
4540                 instr->meter.idx_val = idx_val;
4541
4542                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4543                 instr->meter.length.n_bits = flength->n_bits;
4544                 instr->meter.length.offset = flength->offset / 8;
4545
4546                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4547                 instr->meter.color_in.n_bits = fcin->n_bits;
4548                 instr->meter.color_in.offset = fcin->offset / 8;
4549
4550                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4551                 instr->meter.color_out.n_bits = fcout->n_bits;
4552                 instr->meter.color_out.offset = fcout->offset / 8;
4553
4554                 return 0;
4555         }
4556
4557         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4558         if (!fidx && !fcin) {
4559                 uint32_t idx_val, color_in_val;
4560
4561                 idx_val = strtoul(idx, &idx, 0);
4562                 CHECK(!idx[0], EINVAL);
4563
4564                 color_in_val = strtoul(color_in, &color_in, 0);
4565                 CHECK(!color_in[0], EINVAL);
4566
4567                 instr->type = INSTR_METER_IMI;
4568                 if (length[0] == 'h')
4569                         instr->type = INSTR_METER_IHI;
4570
4571                 instr->meter.metarray_id = m->id;
4572
4573                 instr->meter.idx_val = idx_val;
4574
4575                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4576                 instr->meter.length.n_bits = flength->n_bits;
4577                 instr->meter.length.offset = flength->offset / 8;
4578
4579                 instr->meter.color_in_val = color_in_val;
4580
4581                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4582                 instr->meter.color_out.n_bits = fcout->n_bits;
4583                 instr->meter.color_out.offset = fcout->offset / 8;
4584
4585                 return 0;
4586         }
4587
4588         CHECK(0, EINVAL);
4589 }
4590
4591 static inline void
4592 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4593 {
4594         struct thread *t = &p->threads[p->thread_id];
4595         struct instruction *ip = t->ip;
4596
4597         /* Structs. */
4598         __instr_metprefetch_h_exec(p, t, ip);
4599
4600         /* Thread. */
4601         thread_ip_inc(p);
4602 }
4603
4604 static inline void
4605 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4606 {
4607         struct thread *t = &p->threads[p->thread_id];
4608         struct instruction *ip = t->ip;
4609
4610         /* Structs. */
4611         __instr_metprefetch_m_exec(p, t, ip);
4612
4613         /* Thread. */
4614         thread_ip_inc(p);
4615 }
4616
4617 static inline void
4618 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4619 {
4620         struct thread *t = &p->threads[p->thread_id];
4621         struct instruction *ip = t->ip;
4622
4623         /* Structs. */
4624         __instr_metprefetch_i_exec(p, t, ip);
4625
4626         /* Thread. */
4627         thread_ip_inc(p);
4628 }
4629
4630 static inline void
4631 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4632 {
4633         struct thread *t = &p->threads[p->thread_id];
4634         struct instruction *ip = t->ip;
4635
4636         /* Structs. */
4637         __instr_meter_hhm_exec(p, t, ip);
4638
4639         /* Thread. */
4640         thread_ip_inc(p);
4641 }
4642
4643 static inline void
4644 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4645 {
4646         struct thread *t = &p->threads[p->thread_id];
4647         struct instruction *ip = t->ip;
4648
4649         /* Structs. */
4650         __instr_meter_hhi_exec(p, t, ip);
4651
4652         /* Thread. */
4653         thread_ip_inc(p);
4654 }
4655
4656 static inline void
4657 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4658 {
4659         struct thread *t = &p->threads[p->thread_id];
4660         struct instruction *ip = t->ip;
4661
4662         /* Structs. */
4663         __instr_meter_hmm_exec(p, t, ip);
4664
4665         /* Thread. */
4666         thread_ip_inc(p);
4667 }
4668
4669 static inline void
4670 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4671 {
4672         struct thread *t = &p->threads[p->thread_id];
4673         struct instruction *ip = t->ip;
4674
4675         /* Structs. */
4676         __instr_meter_hmi_exec(p, t, ip);
4677
4678         /* Thread. */
4679         thread_ip_inc(p);
4680 }
4681
4682 static inline void
4683 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4684 {
4685         struct thread *t = &p->threads[p->thread_id];
4686         struct instruction *ip = t->ip;
4687
4688         /* Structs. */
4689         __instr_meter_mhm_exec(p, t, ip);
4690
4691         /* Thread. */
4692         thread_ip_inc(p);
4693 }
4694
4695 static inline void
4696 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4697 {
4698         struct thread *t = &p->threads[p->thread_id];
4699         struct instruction *ip = t->ip;
4700
4701         /* Structs. */
4702         __instr_meter_mhi_exec(p, t, ip);
4703
4704         /* Thread. */
4705         thread_ip_inc(p);
4706 }
4707
4708 static inline void
4709 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4710 {
4711         struct thread *t = &p->threads[p->thread_id];
4712         struct instruction *ip = t->ip;
4713
4714         /* Structs. */
4715         __instr_meter_mmm_exec(p, t, ip);
4716
4717         /* Thread. */
4718         thread_ip_inc(p);
4719 }
4720
4721 static inline void
4722 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4723 {
4724         struct thread *t = &p->threads[p->thread_id];
4725         struct instruction *ip = t->ip;
4726
4727         /* Structs. */
4728         __instr_meter_mmi_exec(p, t, ip);
4729
4730         /* Thread. */
4731         thread_ip_inc(p);
4732 }
4733
4734 static inline void
4735 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4736 {
4737         struct thread *t = &p->threads[p->thread_id];
4738         struct instruction *ip = t->ip;
4739
4740         /* Structs. */
4741         __instr_meter_ihm_exec(p, t, ip);
4742
4743         /* Thread. */
4744         thread_ip_inc(p);
4745 }
4746
4747 static inline void
4748 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4749 {
4750         struct thread *t = &p->threads[p->thread_id];
4751         struct instruction *ip = t->ip;
4752
4753         /* Structs. */
4754         __instr_meter_ihi_exec(p, t, ip);
4755
4756         /* Thread. */
4757         thread_ip_inc(p);
4758 }
4759
4760 static inline void
4761 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4762 {
4763         struct thread *t = &p->threads[p->thread_id];
4764         struct instruction *ip = t->ip;
4765
4766         /* Structs. */
4767         __instr_meter_imm_exec(p, t, ip);
4768
4769         /* Thread. */
4770         thread_ip_inc(p);
4771 }
4772
4773 static inline void
4774 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4775 {
4776         struct thread *t = &p->threads[p->thread_id];
4777         struct instruction *ip = t->ip;
4778
4779         /* Structs. */
4780         __instr_meter_imi_exec(p, t, ip);
4781
4782         /* Thread. */
4783         thread_ip_inc(p);
4784 }
4785
4786 /*
4787  * jmp.
4788  */
4789 static int
4790 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4791                     struct action *action __rte_unused,
4792                     char **tokens,
4793                     int n_tokens,
4794                     struct instruction *instr,
4795                     struct instruction_data *data)
4796 {
4797         CHECK(n_tokens == 2, EINVAL);
4798
4799         strcpy(data->jmp_label, tokens[1]);
4800
4801         instr->type = INSTR_JMP;
4802         instr->jmp.ip = NULL; /* Resolved later. */
4803         return 0;
4804 }
4805
4806 static int
4807 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4808                           struct action *action __rte_unused,
4809                           char **tokens,
4810                           int n_tokens,
4811                           struct instruction *instr,
4812                           struct instruction_data *data)
4813 {
4814         struct header *h;
4815
4816         CHECK(n_tokens == 3, EINVAL);
4817
4818         strcpy(data->jmp_label, tokens[1]);
4819
4820         h = header_parse(p, tokens[2]);
4821         CHECK(h, EINVAL);
4822
4823         instr->type = INSTR_JMP_VALID;
4824         instr->jmp.ip = NULL; /* Resolved later. */
4825         instr->jmp.header_id = h->id;
4826         return 0;
4827 }
4828
4829 static int
4830 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4831                             struct action *action __rte_unused,
4832                             char **tokens,
4833                             int n_tokens,
4834                             struct instruction *instr,
4835                             struct instruction_data *data)
4836 {
4837         struct header *h;
4838
4839         CHECK(n_tokens == 3, EINVAL);
4840
4841         strcpy(data->jmp_label, tokens[1]);
4842
4843         h = header_parse(p, tokens[2]);
4844         CHECK(h, EINVAL);
4845
4846         instr->type = INSTR_JMP_INVALID;
4847         instr->jmp.ip = NULL; /* Resolved later. */
4848         instr->jmp.header_id = h->id;
4849         return 0;
4850 }
4851
4852 static int
4853 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4854                         struct action *action,
4855                         char **tokens,
4856                         int n_tokens,
4857                         struct instruction *instr,
4858                         struct instruction_data *data)
4859 {
4860         CHECK(!action, EINVAL);
4861         CHECK(n_tokens == 2, EINVAL);
4862
4863         strcpy(data->jmp_label, tokens[1]);
4864
4865         instr->type = INSTR_JMP_HIT;
4866         instr->jmp.ip = NULL; /* Resolved later. */
4867         return 0;
4868 }
4869
4870 static int
4871 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4872                          struct action *action,
4873                          char **tokens,
4874                          int n_tokens,
4875                          struct instruction *instr,
4876                          struct instruction_data *data)
4877 {
4878         CHECK(!action, EINVAL);
4879         CHECK(n_tokens == 2, EINVAL);
4880
4881         strcpy(data->jmp_label, tokens[1]);
4882
4883         instr->type = INSTR_JMP_MISS;
4884         instr->jmp.ip = NULL; /* Resolved later. */
4885         return 0;
4886 }
4887
4888 static int
4889 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4890                                struct action *action,
4891                                char **tokens,
4892                                int n_tokens,
4893                                struct instruction *instr,
4894                                struct instruction_data *data)
4895 {
4896         struct action *a;
4897
4898         CHECK(!action, EINVAL);
4899         CHECK(n_tokens == 3, EINVAL);
4900
4901         strcpy(data->jmp_label, tokens[1]);
4902
4903         a = action_find(p, tokens[2]);
4904         CHECK(a, EINVAL);
4905
4906         instr->type = INSTR_JMP_ACTION_HIT;
4907         instr->jmp.ip = NULL; /* Resolved later. */
4908         instr->jmp.action_id = a->id;
4909         return 0;
4910 }
4911
4912 static int
4913 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4914                                 struct action *action,
4915                                 char **tokens,
4916                                 int n_tokens,
4917                                 struct instruction *instr,
4918                                 struct instruction_data *data)
4919 {
4920         struct action *a;
4921
4922         CHECK(!action, EINVAL);
4923         CHECK(n_tokens == 3, EINVAL);
4924
4925         strcpy(data->jmp_label, tokens[1]);
4926
4927         a = action_find(p, tokens[2]);
4928         CHECK(a, EINVAL);
4929
4930         instr->type = INSTR_JMP_ACTION_MISS;
4931         instr->jmp.ip = NULL; /* Resolved later. */
4932         instr->jmp.action_id = a->id;
4933         return 0;
4934 }
4935
4936 static int
4937 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4938                        struct action *action,
4939                        char **tokens,
4940                        int n_tokens,
4941                        struct instruction *instr,
4942                        struct instruction_data *data)
4943 {
4944         char *a = tokens[2], *b = tokens[3];
4945         struct field *fa, *fb;
4946         uint64_t b_val;
4947         uint32_t a_struct_id, b_struct_id;
4948
4949         CHECK(n_tokens == 4, EINVAL);
4950
4951         strcpy(data->jmp_label, tokens[1]);
4952
4953         fa = struct_field_parse(p, action, a, &a_struct_id);
4954         CHECK(fa, EINVAL);
4955         CHECK(!fa->var_size, EINVAL);
4956
4957         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
4958         fb = struct_field_parse(p, action, b, &b_struct_id);
4959         if (fb) {
4960                 CHECK(!fb->var_size, EINVAL);
4961
4962                 instr->type = INSTR_JMP_EQ;
4963                 if (a[0] != 'h' && b[0] == 'h')
4964                         instr->type = INSTR_JMP_EQ_MH;
4965                 if (a[0] == 'h' && b[0] != 'h')
4966                         instr->type = INSTR_JMP_EQ_HM;
4967                 if (a[0] == 'h' && b[0] == 'h')
4968                         instr->type = INSTR_JMP_EQ_HH;
4969                 instr->jmp.ip = NULL; /* Resolved later. */
4970
4971                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4972                 instr->jmp.a.n_bits = fa->n_bits;
4973                 instr->jmp.a.offset = fa->offset / 8;
4974                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4975                 instr->jmp.b.n_bits = fb->n_bits;
4976                 instr->jmp.b.offset = fb->offset / 8;
4977                 return 0;
4978         }
4979
4980         /* JMP_EQ_I. */
4981         b_val = strtoull(b, &b, 0);
4982         CHECK(!b[0], EINVAL);
4983
4984         if (a[0] == 'h')
4985                 b_val = hton64(b_val) >> (64 - fa->n_bits);
4986
4987         instr->type = INSTR_JMP_EQ_I;
4988         instr->jmp.ip = NULL; /* Resolved later. */
4989         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4990         instr->jmp.a.n_bits = fa->n_bits;
4991         instr->jmp.a.offset = fa->offset / 8;
4992         instr->jmp.b_val = b_val;
4993         return 0;
4994 }
4995
4996 static int
4997 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
4998                         struct action *action,
4999                         char **tokens,
5000                         int n_tokens,
5001                         struct instruction *instr,
5002                         struct instruction_data *data)
5003 {
5004         char *a = tokens[2], *b = tokens[3];
5005         struct field *fa, *fb;
5006         uint64_t b_val;
5007         uint32_t a_struct_id, b_struct_id;
5008
5009         CHECK(n_tokens == 4, EINVAL);
5010
5011         strcpy(data->jmp_label, tokens[1]);
5012
5013         fa = struct_field_parse(p, action, a, &a_struct_id);
5014         CHECK(fa, EINVAL);
5015         CHECK(!fa->var_size, EINVAL);
5016
5017         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5018         fb = struct_field_parse(p, action, b, &b_struct_id);
5019         if (fb) {
5020                 CHECK(!fb->var_size, EINVAL);
5021
5022                 instr->type = INSTR_JMP_NEQ;
5023                 if (a[0] != 'h' && b[0] == 'h')
5024                         instr->type = INSTR_JMP_NEQ_MH;
5025                 if (a[0] == 'h' && b[0] != 'h')
5026                         instr->type = INSTR_JMP_NEQ_HM;
5027                 if (a[0] == 'h' && b[0] == 'h')
5028                         instr->type = INSTR_JMP_NEQ_HH;
5029                 instr->jmp.ip = NULL; /* Resolved later. */
5030
5031                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5032                 instr->jmp.a.n_bits = fa->n_bits;
5033                 instr->jmp.a.offset = fa->offset / 8;
5034                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5035                 instr->jmp.b.n_bits = fb->n_bits;
5036                 instr->jmp.b.offset = fb->offset / 8;
5037                 return 0;
5038         }
5039
5040         /* JMP_NEQ_I. */
5041         b_val = strtoull(b, &b, 0);
5042         CHECK(!b[0], EINVAL);
5043
5044         if (a[0] == 'h')
5045                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5046
5047         instr->type = INSTR_JMP_NEQ_I;
5048         instr->jmp.ip = NULL; /* Resolved later. */
5049         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5050         instr->jmp.a.n_bits = fa->n_bits;
5051         instr->jmp.a.offset = fa->offset / 8;
5052         instr->jmp.b_val = b_val;
5053         return 0;
5054 }
5055
5056 static int
5057 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5058                        struct action *action,
5059                        char **tokens,
5060                        int n_tokens,
5061                        struct instruction *instr,
5062                        struct instruction_data *data)
5063 {
5064         char *a = tokens[2], *b = tokens[3];
5065         struct field *fa, *fb;
5066         uint64_t b_val;
5067         uint32_t a_struct_id, b_struct_id;
5068
5069         CHECK(n_tokens == 4, EINVAL);
5070
5071         strcpy(data->jmp_label, tokens[1]);
5072
5073         fa = struct_field_parse(p, action, a, &a_struct_id);
5074         CHECK(fa, EINVAL);
5075         CHECK(!fa->var_size, EINVAL);
5076
5077         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5078         fb = struct_field_parse(p, action, b, &b_struct_id);
5079         if (fb) {
5080                 CHECK(!fb->var_size, EINVAL);
5081
5082                 instr->type = INSTR_JMP_LT;
5083                 if (a[0] == 'h' && b[0] != 'h')
5084                         instr->type = INSTR_JMP_LT_HM;
5085                 if (a[0] != 'h' && b[0] == 'h')
5086                         instr->type = INSTR_JMP_LT_MH;
5087                 if (a[0] == 'h' && b[0] == 'h')
5088                         instr->type = INSTR_JMP_LT_HH;
5089                 instr->jmp.ip = NULL; /* Resolved later. */
5090
5091                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5092                 instr->jmp.a.n_bits = fa->n_bits;
5093                 instr->jmp.a.offset = fa->offset / 8;
5094                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5095                 instr->jmp.b.n_bits = fb->n_bits;
5096                 instr->jmp.b.offset = fb->offset / 8;
5097                 return 0;
5098         }
5099
5100         /* JMP_LT_MI, JMP_LT_HI. */
5101         b_val = strtoull(b, &b, 0);
5102         CHECK(!b[0], EINVAL);
5103
5104         instr->type = INSTR_JMP_LT_MI;
5105         if (a[0] == 'h')
5106                 instr->type = INSTR_JMP_LT_HI;
5107         instr->jmp.ip = NULL; /* Resolved later. */
5108
5109         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5110         instr->jmp.a.n_bits = fa->n_bits;
5111         instr->jmp.a.offset = fa->offset / 8;
5112         instr->jmp.b_val = b_val;
5113         return 0;
5114 }
5115
5116 static int
5117 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5118                        struct action *action,
5119                        char **tokens,
5120                        int n_tokens,
5121                        struct instruction *instr,
5122                        struct instruction_data *data)
5123 {
5124         char *a = tokens[2], *b = tokens[3];
5125         struct field *fa, *fb;
5126         uint64_t b_val;
5127         uint32_t a_struct_id, b_struct_id;
5128
5129         CHECK(n_tokens == 4, EINVAL);
5130
5131         strcpy(data->jmp_label, tokens[1]);
5132
5133         fa = struct_field_parse(p, action, a, &a_struct_id);
5134         CHECK(fa, EINVAL);
5135         CHECK(!fa->var_size, EINVAL);
5136
5137         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5138         fb = struct_field_parse(p, action, b, &b_struct_id);
5139         if (fb) {
5140                 CHECK(!fb->var_size, EINVAL);
5141
5142                 instr->type = INSTR_JMP_GT;
5143                 if (a[0] == 'h' && b[0] != 'h')
5144                         instr->type = INSTR_JMP_GT_HM;
5145                 if (a[0] != 'h' && b[0] == 'h')
5146                         instr->type = INSTR_JMP_GT_MH;
5147                 if (a[0] == 'h' && b[0] == 'h')
5148                         instr->type = INSTR_JMP_GT_HH;
5149                 instr->jmp.ip = NULL; /* Resolved later. */
5150
5151                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5152                 instr->jmp.a.n_bits = fa->n_bits;
5153                 instr->jmp.a.offset = fa->offset / 8;
5154                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5155                 instr->jmp.b.n_bits = fb->n_bits;
5156                 instr->jmp.b.offset = fb->offset / 8;
5157                 return 0;
5158         }
5159
5160         /* JMP_GT_MI, JMP_GT_HI. */
5161         b_val = strtoull(b, &b, 0);
5162         CHECK(!b[0], EINVAL);
5163
5164         instr->type = INSTR_JMP_GT_MI;
5165         if (a[0] == 'h')
5166                 instr->type = INSTR_JMP_GT_HI;
5167         instr->jmp.ip = NULL; /* Resolved later. */
5168
5169         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5170         instr->jmp.a.n_bits = fa->n_bits;
5171         instr->jmp.a.offset = fa->offset / 8;
5172         instr->jmp.b_val = b_val;
5173         return 0;
5174 }
5175
5176 static inline void
5177 instr_jmp_exec(struct rte_swx_pipeline *p)
5178 {
5179         struct thread *t = &p->threads[p->thread_id];
5180         struct instruction *ip = t->ip;
5181
5182         TRACE("[Thread %2u] jmp\n", p->thread_id);
5183
5184         thread_ip_set(t, ip->jmp.ip);
5185 }
5186
5187 static inline void
5188 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5189 {
5190         struct thread *t = &p->threads[p->thread_id];
5191         struct instruction *ip = t->ip;
5192         uint32_t header_id = ip->jmp.header_id;
5193
5194         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5195
5196         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5197 }
5198
5199 static inline void
5200 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5201 {
5202         struct thread *t = &p->threads[p->thread_id];
5203         struct instruction *ip = t->ip;
5204         uint32_t header_id = ip->jmp.header_id;
5205
5206         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5207
5208         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5209 }
5210
5211 static inline void
5212 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5213 {
5214         struct thread *t = &p->threads[p->thread_id];
5215         struct instruction *ip = t->ip;
5216         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5217
5218         TRACE("[Thread %2u] jmph\n", p->thread_id);
5219
5220         t->ip = ip_next[t->hit];
5221 }
5222
5223 static inline void
5224 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5225 {
5226         struct thread *t = &p->threads[p->thread_id];
5227         struct instruction *ip = t->ip;
5228         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5229
5230         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5231
5232         t->ip = ip_next[t->hit];
5233 }
5234
5235 static inline void
5236 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5237 {
5238         struct thread *t = &p->threads[p->thread_id];
5239         struct instruction *ip = t->ip;
5240
5241         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5242
5243         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5244 }
5245
5246 static inline void
5247 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5248 {
5249         struct thread *t = &p->threads[p->thread_id];
5250         struct instruction *ip = t->ip;
5251
5252         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5253
5254         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5255 }
5256
5257 static inline void
5258 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5259 {
5260         struct thread *t = &p->threads[p->thread_id];
5261         struct instruction *ip = t->ip;
5262
5263         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5264
5265         JMP_CMP(t, ip, ==);
5266 }
5267
5268 static inline void
5269 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5270 {
5271         struct thread *t = &p->threads[p->thread_id];
5272         struct instruction *ip = t->ip;
5273
5274         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5275
5276         JMP_CMP_MH(t, ip, ==);
5277 }
5278
5279 static inline void
5280 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5281 {
5282         struct thread *t = &p->threads[p->thread_id];
5283         struct instruction *ip = t->ip;
5284
5285         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5286
5287         JMP_CMP_HM(t, ip, ==);
5288 }
5289
5290 static inline void
5291 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5292 {
5293         struct thread *t = &p->threads[p->thread_id];
5294         struct instruction *ip = t->ip;
5295
5296         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5297
5298         JMP_CMP_HH_FAST(t, ip, ==);
5299 }
5300
5301 static inline void
5302 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5303 {
5304         struct thread *t = &p->threads[p->thread_id];
5305         struct instruction *ip = t->ip;
5306
5307         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5308
5309         JMP_CMP_I(t, ip, ==);
5310 }
5311
5312 static inline void
5313 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5314 {
5315         struct thread *t = &p->threads[p->thread_id];
5316         struct instruction *ip = t->ip;
5317
5318         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5319
5320         JMP_CMP(t, ip, !=);
5321 }
5322
5323 static inline void
5324 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5325 {
5326         struct thread *t = &p->threads[p->thread_id];
5327         struct instruction *ip = t->ip;
5328
5329         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5330
5331         JMP_CMP_MH(t, ip, !=);
5332 }
5333
5334 static inline void
5335 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5336 {
5337         struct thread *t = &p->threads[p->thread_id];
5338         struct instruction *ip = t->ip;
5339
5340         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5341
5342         JMP_CMP_HM(t, ip, !=);
5343 }
5344
5345 static inline void
5346 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5347 {
5348         struct thread *t = &p->threads[p->thread_id];
5349         struct instruction *ip = t->ip;
5350
5351         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5352
5353         JMP_CMP_HH_FAST(t, ip, !=);
5354 }
5355
5356 static inline void
5357 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5358 {
5359         struct thread *t = &p->threads[p->thread_id];
5360         struct instruction *ip = t->ip;
5361
5362         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5363
5364         JMP_CMP_I(t, ip, !=);
5365 }
5366
5367 static inline void
5368 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5369 {
5370         struct thread *t = &p->threads[p->thread_id];
5371         struct instruction *ip = t->ip;
5372
5373         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5374
5375         JMP_CMP(t, ip, <);
5376 }
5377
5378 static inline void
5379 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5380 {
5381         struct thread *t = &p->threads[p->thread_id];
5382         struct instruction *ip = t->ip;
5383
5384         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5385
5386         JMP_CMP_MH(t, ip, <);
5387 }
5388
5389 static inline void
5390 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5391 {
5392         struct thread *t = &p->threads[p->thread_id];
5393         struct instruction *ip = t->ip;
5394
5395         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5396
5397         JMP_CMP_HM(t, ip, <);
5398 }
5399
5400 static inline void
5401 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5402 {
5403         struct thread *t = &p->threads[p->thread_id];
5404         struct instruction *ip = t->ip;
5405
5406         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5407
5408         JMP_CMP_HH(t, ip, <);
5409 }
5410
5411 static inline void
5412 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5413 {
5414         struct thread *t = &p->threads[p->thread_id];
5415         struct instruction *ip = t->ip;
5416
5417         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5418
5419         JMP_CMP_MI(t, ip, <);
5420 }
5421
5422 static inline void
5423 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5424 {
5425         struct thread *t = &p->threads[p->thread_id];
5426         struct instruction *ip = t->ip;
5427
5428         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5429
5430         JMP_CMP_HI(t, ip, <);
5431 }
5432
5433 static inline void
5434 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5435 {
5436         struct thread *t = &p->threads[p->thread_id];
5437         struct instruction *ip = t->ip;
5438
5439         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5440
5441         JMP_CMP(t, ip, >);
5442 }
5443
5444 static inline void
5445 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5446 {
5447         struct thread *t = &p->threads[p->thread_id];
5448         struct instruction *ip = t->ip;
5449
5450         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5451
5452         JMP_CMP_MH(t, ip, >);
5453 }
5454
5455 static inline void
5456 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5457 {
5458         struct thread *t = &p->threads[p->thread_id];
5459         struct instruction *ip = t->ip;
5460
5461         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5462
5463         JMP_CMP_HM(t, ip, >);
5464 }
5465
5466 static inline void
5467 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5468 {
5469         struct thread *t = &p->threads[p->thread_id];
5470         struct instruction *ip = t->ip;
5471
5472         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5473
5474         JMP_CMP_HH(t, ip, >);
5475 }
5476
5477 static inline void
5478 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5479 {
5480         struct thread *t = &p->threads[p->thread_id];
5481         struct instruction *ip = t->ip;
5482
5483         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5484
5485         JMP_CMP_MI(t, ip, >);
5486 }
5487
5488 static inline void
5489 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5490 {
5491         struct thread *t = &p->threads[p->thread_id];
5492         struct instruction *ip = t->ip;
5493
5494         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5495
5496         JMP_CMP_HI(t, ip, >);
5497 }
5498
5499 /*
5500  * return.
5501  */
5502 static int
5503 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5504                        struct action *action,
5505                        char **tokens __rte_unused,
5506                        int n_tokens,
5507                        struct instruction *instr,
5508                        struct instruction_data *data __rte_unused)
5509 {
5510         CHECK(action, EINVAL);
5511         CHECK(n_tokens == 1, EINVAL);
5512
5513         instr->type = INSTR_RETURN;
5514         return 0;
5515 }
5516
5517 static inline void
5518 instr_return_exec(struct rte_swx_pipeline *p)
5519 {
5520         struct thread *t = &p->threads[p->thread_id];
5521
5522         TRACE("[Thread %2u] return\n", p->thread_id);
5523
5524         t->ip = t->ret;
5525 }
5526
5527 static int
5528 instr_translate(struct rte_swx_pipeline *p,
5529                 struct action *action,
5530                 char *string,
5531                 struct instruction *instr,
5532                 struct instruction_data *data)
5533 {
5534         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5535         int n_tokens = 0, tpos = 0;
5536
5537         /* Parse the instruction string into tokens. */
5538         for ( ; ; ) {
5539                 char *token;
5540
5541                 token = strtok_r(string, " \t\v", &string);
5542                 if (!token)
5543                         break;
5544
5545                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5546                 CHECK_NAME(token, EINVAL);
5547
5548                 tokens[n_tokens] = token;
5549                 n_tokens++;
5550         }
5551
5552         CHECK(n_tokens, EINVAL);
5553
5554         /* Handle the optional instruction label. */
5555         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5556                 strcpy(data->label, tokens[0]);
5557
5558                 tpos += 2;
5559                 CHECK(n_tokens - tpos, EINVAL);
5560         }
5561
5562         /* Identify the instruction type. */
5563         if (!strcmp(tokens[tpos], "rx"))
5564                 return instr_rx_translate(p,
5565                                           action,
5566                                           &tokens[tpos],
5567                                           n_tokens - tpos,
5568                                           instr,
5569                                           data);
5570
5571         if (!strcmp(tokens[tpos], "tx"))
5572                 return instr_tx_translate(p,
5573                                           action,
5574                                           &tokens[tpos],
5575                                           n_tokens - tpos,
5576                                           instr,
5577                                           data);
5578
5579         if (!strcmp(tokens[tpos], "drop"))
5580                 return instr_drop_translate(p,
5581                                             action,
5582                                             &tokens[tpos],
5583                                             n_tokens - tpos,
5584                                             instr,
5585                                             data);
5586
5587         if (!strcmp(tokens[tpos], "extract"))
5588                 return instr_hdr_extract_translate(p,
5589                                                    action,
5590                                                    &tokens[tpos],
5591                                                    n_tokens - tpos,
5592                                                    instr,
5593                                                    data);
5594
5595         if (!strcmp(tokens[tpos], "lookahead"))
5596                 return instr_hdr_lookahead_translate(p,
5597                                                      action,
5598                                                      &tokens[tpos],
5599                                                      n_tokens - tpos,
5600                                                      instr,
5601                                                      data);
5602
5603         if (!strcmp(tokens[tpos], "emit"))
5604                 return instr_hdr_emit_translate(p,
5605                                                 action,
5606                                                 &tokens[tpos],
5607                                                 n_tokens - tpos,
5608                                                 instr,
5609                                                 data);
5610
5611         if (!strcmp(tokens[tpos], "validate"))
5612                 return instr_hdr_validate_translate(p,
5613                                                     action,
5614                                                     &tokens[tpos],
5615                                                     n_tokens - tpos,
5616                                                     instr,
5617                                                     data);
5618
5619         if (!strcmp(tokens[tpos], "invalidate"))
5620                 return instr_hdr_invalidate_translate(p,
5621                                                       action,
5622                                                       &tokens[tpos],
5623                                                       n_tokens - tpos,
5624                                                       instr,
5625                                                       data);
5626
5627         if (!strcmp(tokens[tpos], "mov"))
5628                 return instr_mov_translate(p,
5629                                            action,
5630                                            &tokens[tpos],
5631                                            n_tokens - tpos,
5632                                            instr,
5633                                            data);
5634
5635         if (!strcmp(tokens[tpos], "add"))
5636                 return instr_alu_add_translate(p,
5637                                                action,
5638                                                &tokens[tpos],
5639                                                n_tokens - tpos,
5640                                                instr,
5641                                                data);
5642
5643         if (!strcmp(tokens[tpos], "sub"))
5644                 return instr_alu_sub_translate(p,
5645                                                action,
5646                                                &tokens[tpos],
5647                                                n_tokens - tpos,
5648                                                instr,
5649                                                data);
5650
5651         if (!strcmp(tokens[tpos], "ckadd"))
5652                 return instr_alu_ckadd_translate(p,
5653                                                  action,
5654                                                  &tokens[tpos],
5655                                                  n_tokens - tpos,
5656                                                  instr,
5657                                                  data);
5658
5659         if (!strcmp(tokens[tpos], "cksub"))
5660                 return instr_alu_cksub_translate(p,
5661                                                  action,
5662                                                  &tokens[tpos],
5663                                                  n_tokens - tpos,
5664                                                  instr,
5665                                                  data);
5666
5667         if (!strcmp(tokens[tpos], "and"))
5668                 return instr_alu_and_translate(p,
5669                                                action,
5670                                                &tokens[tpos],
5671                                                n_tokens - tpos,
5672                                                instr,
5673                                                data);
5674
5675         if (!strcmp(tokens[tpos], "or"))
5676                 return instr_alu_or_translate(p,
5677                                               action,
5678                                               &tokens[tpos],
5679                                               n_tokens - tpos,
5680                                               instr,
5681                                               data);
5682
5683         if (!strcmp(tokens[tpos], "xor"))
5684                 return instr_alu_xor_translate(p,
5685                                                action,
5686                                                &tokens[tpos],
5687                                                n_tokens - tpos,
5688                                                instr,
5689                                                data);
5690
5691         if (!strcmp(tokens[tpos], "shl"))
5692                 return instr_alu_shl_translate(p,
5693                                                action,
5694                                                &tokens[tpos],
5695                                                n_tokens - tpos,
5696                                                instr,
5697                                                data);
5698
5699         if (!strcmp(tokens[tpos], "shr"))
5700                 return instr_alu_shr_translate(p,
5701                                                action,
5702                                                &tokens[tpos],
5703                                                n_tokens - tpos,
5704                                                instr,
5705                                                data);
5706
5707         if (!strcmp(tokens[tpos], "regprefetch"))
5708                 return instr_regprefetch_translate(p,
5709                                                    action,
5710                                                    &tokens[tpos],
5711                                                    n_tokens - tpos,
5712                                                    instr,
5713                                                    data);
5714
5715         if (!strcmp(tokens[tpos], "regrd"))
5716                 return instr_regrd_translate(p,
5717                                              action,
5718                                              &tokens[tpos],
5719                                              n_tokens - tpos,
5720                                              instr,
5721                                              data);
5722
5723         if (!strcmp(tokens[tpos], "regwr"))
5724                 return instr_regwr_translate(p,
5725                                              action,
5726                                              &tokens[tpos],
5727                                              n_tokens - tpos,
5728                                              instr,
5729                                              data);
5730
5731         if (!strcmp(tokens[tpos], "regadd"))
5732                 return instr_regadd_translate(p,
5733                                               action,
5734                                               &tokens[tpos],
5735                                               n_tokens - tpos,
5736                                               instr,
5737                                               data);
5738
5739         if (!strcmp(tokens[tpos], "metprefetch"))
5740                 return instr_metprefetch_translate(p,
5741                                                    action,
5742                                                    &tokens[tpos],
5743                                                    n_tokens - tpos,
5744                                                    instr,
5745                                                    data);
5746
5747         if (!strcmp(tokens[tpos], "meter"))
5748                 return instr_meter_translate(p,
5749                                              action,
5750                                              &tokens[tpos],
5751                                              n_tokens - tpos,
5752                                              instr,
5753                                              data);
5754
5755         if (!strcmp(tokens[tpos], "table"))
5756                 return instr_table_translate(p,
5757                                              action,
5758                                              &tokens[tpos],
5759                                              n_tokens - tpos,
5760                                              instr,
5761                                              data);
5762
5763         if (!strcmp(tokens[tpos], "learn"))
5764                 return instr_learn_translate(p,
5765                                              action,
5766                                              &tokens[tpos],
5767                                              n_tokens - tpos,
5768                                              instr,
5769                                              data);
5770
5771         if (!strcmp(tokens[tpos], "forget"))
5772                 return instr_forget_translate(p,
5773                                               action,
5774                                               &tokens[tpos],
5775                                               n_tokens - tpos,
5776                                               instr,
5777                                               data);
5778
5779         if (!strcmp(tokens[tpos], "extern"))
5780                 return instr_extern_translate(p,
5781                                               action,
5782                                               &tokens[tpos],
5783                                               n_tokens - tpos,
5784                                               instr,
5785                                               data);
5786
5787         if (!strcmp(tokens[tpos], "jmp"))
5788                 return instr_jmp_translate(p,
5789                                            action,
5790                                            &tokens[tpos],
5791                                            n_tokens - tpos,
5792                                            instr,
5793                                            data);
5794
5795         if (!strcmp(tokens[tpos], "jmpv"))
5796                 return instr_jmp_valid_translate(p,
5797                                                  action,
5798                                                  &tokens[tpos],
5799                                                  n_tokens - tpos,
5800                                                  instr,
5801                                                  data);
5802
5803         if (!strcmp(tokens[tpos], "jmpnv"))
5804                 return instr_jmp_invalid_translate(p,
5805                                                    action,
5806                                                    &tokens[tpos],
5807                                                    n_tokens - tpos,
5808                                                    instr,
5809                                                    data);
5810
5811         if (!strcmp(tokens[tpos], "jmph"))
5812                 return instr_jmp_hit_translate(p,
5813                                                action,
5814                                                &tokens[tpos],
5815                                                n_tokens - tpos,
5816                                                instr,
5817                                                data);
5818
5819         if (!strcmp(tokens[tpos], "jmpnh"))
5820                 return instr_jmp_miss_translate(p,
5821                                                 action,
5822                                                 &tokens[tpos],
5823                                                 n_tokens - tpos,
5824                                                 instr,
5825                                                 data);
5826
5827         if (!strcmp(tokens[tpos], "jmpa"))
5828                 return instr_jmp_action_hit_translate(p,
5829                                                       action,
5830                                                       &tokens[tpos],
5831                                                       n_tokens - tpos,
5832                                                       instr,
5833                                                       data);
5834
5835         if (!strcmp(tokens[tpos], "jmpna"))
5836                 return instr_jmp_action_miss_translate(p,
5837                                                        action,
5838                                                        &tokens[tpos],
5839                                                        n_tokens - tpos,
5840                                                        instr,
5841                                                        data);
5842
5843         if (!strcmp(tokens[tpos], "jmpeq"))
5844                 return instr_jmp_eq_translate(p,
5845                                               action,
5846                                               &tokens[tpos],
5847                                               n_tokens - tpos,
5848                                               instr,
5849                                               data);
5850
5851         if (!strcmp(tokens[tpos], "jmpneq"))
5852                 return instr_jmp_neq_translate(p,
5853                                                action,
5854                                                &tokens[tpos],
5855                                                n_tokens - tpos,
5856                                                instr,
5857                                                data);
5858
5859         if (!strcmp(tokens[tpos], "jmplt"))
5860                 return instr_jmp_lt_translate(p,
5861                                               action,
5862                                               &tokens[tpos],
5863                                               n_tokens - tpos,
5864                                               instr,
5865                                               data);
5866
5867         if (!strcmp(tokens[tpos], "jmpgt"))
5868                 return instr_jmp_gt_translate(p,
5869                                               action,
5870                                               &tokens[tpos],
5871                                               n_tokens - tpos,
5872                                               instr,
5873                                               data);
5874
5875         if (!strcmp(tokens[tpos], "return"))
5876                 return instr_return_translate(p,
5877                                               action,
5878                                               &tokens[tpos],
5879                                               n_tokens - tpos,
5880                                               instr,
5881                                               data);
5882
5883         CHECK(0, EINVAL);
5884 }
5885
5886 static struct instruction_data *
5887 label_find(struct instruction_data *data, uint32_t n, const char *label)
5888 {
5889         uint32_t i;
5890
5891         for (i = 0; i < n; i++)
5892                 if (!strcmp(label, data[i].label))
5893                         return &data[i];
5894
5895         return NULL;
5896 }
5897
5898 static uint32_t
5899 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5900 {
5901         uint32_t count = 0, i;
5902
5903         if (!label[0])
5904                 return 0;
5905
5906         for (i = 0; i < n; i++)
5907                 if (!strcmp(label, data[i].jmp_label))
5908                         count++;
5909
5910         return count;
5911 }
5912
5913 static int
5914 instr_label_check(struct instruction_data *instruction_data,
5915                   uint32_t n_instructions)
5916 {
5917         uint32_t i;
5918
5919         /* Check that all instruction labels are unique. */
5920         for (i = 0; i < n_instructions; i++) {
5921                 struct instruction_data *data = &instruction_data[i];
5922                 char *label = data->label;
5923                 uint32_t j;
5924
5925                 if (!label[0])
5926                         continue;
5927
5928                 for (j = i + 1; j < n_instructions; j++)
5929                         CHECK(strcmp(label, data[j].label), EINVAL);
5930         }
5931
5932         /* Get users for each instruction label. */
5933         for (i = 0; i < n_instructions; i++) {
5934                 struct instruction_data *data = &instruction_data[i];
5935                 char *label = data->label;
5936
5937                 data->n_users = label_is_used(instruction_data,
5938                                               n_instructions,
5939                                               label);
5940         }
5941
5942         return 0;
5943 }
5944
5945 static int
5946 instr_jmp_resolve(struct instruction *instructions,
5947                   struct instruction_data *instruction_data,
5948                   uint32_t n_instructions)
5949 {
5950         uint32_t i;
5951
5952         for (i = 0; i < n_instructions; i++) {
5953                 struct instruction *instr = &instructions[i];
5954                 struct instruction_data *data = &instruction_data[i];
5955                 struct instruction_data *found;
5956
5957                 if (!instruction_is_jmp(instr))
5958                         continue;
5959
5960                 found = label_find(instruction_data,
5961                                    n_instructions,
5962                                    data->jmp_label);
5963                 CHECK(found, EINVAL);
5964
5965                 instr->jmp.ip = &instructions[found - instruction_data];
5966         }
5967
5968         return 0;
5969 }
5970
5971 static int
5972 instr_verify(struct rte_swx_pipeline *p __rte_unused,
5973              struct action *a,
5974              struct instruction *instr,
5975              struct instruction_data *data __rte_unused,
5976              uint32_t n_instructions)
5977 {
5978         if (!a) {
5979                 enum instruction_type type;
5980                 uint32_t i;
5981
5982                 /* Check that the first instruction is rx. */
5983                 CHECK(instr[0].type == INSTR_RX, EINVAL);
5984
5985                 /* Check that there is at least one tx instruction. */
5986                 for (i = 0; i < n_instructions; i++) {
5987                         type = instr[i].type;
5988
5989                         if (instruction_is_tx(type))
5990                                 break;
5991                 }
5992                 CHECK(i < n_instructions, EINVAL);
5993
5994                 /* Check that the last instruction is either tx or unconditional
5995                  * jump.
5996                  */
5997                 type = instr[n_instructions - 1].type;
5998                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
5999         }
6000
6001         if (a) {
6002                 enum instruction_type type;
6003                 uint32_t i;
6004
6005                 /* Check that there is at least one return or tx instruction. */
6006                 for (i = 0; i < n_instructions; i++) {
6007                         type = instr[i].type;
6008
6009                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6010                                 break;
6011                 }
6012                 CHECK(i < n_instructions, EINVAL);
6013         }
6014
6015         return 0;
6016 }
6017
6018 static uint32_t
6019 instr_compact(struct instruction *instructions,
6020               struct instruction_data *instruction_data,
6021               uint32_t n_instructions)
6022 {
6023         uint32_t i, pos = 0;
6024
6025         /* Eliminate the invalid instructions that have been optimized out. */
6026         for (i = 0; i < n_instructions; i++) {
6027                 struct instruction *instr = &instructions[i];
6028                 struct instruction_data *data = &instruction_data[i];
6029
6030                 if (data->invalid)
6031                         continue;
6032
6033                 if (i != pos) {
6034                         memcpy(&instructions[pos], instr, sizeof(*instr));
6035                         memcpy(&instruction_data[pos], data, sizeof(*data));
6036                 }
6037
6038                 pos++;
6039         }
6040
6041         return pos;
6042 }
6043
6044 static int
6045 instr_pattern_extract_many_search(struct instruction *instr,
6046                                   struct instruction_data *data,
6047                                   uint32_t n_instr,
6048                                   uint32_t *n_pattern_instr)
6049 {
6050         uint32_t i;
6051
6052         for (i = 0; i < n_instr; i++) {
6053                 if (data[i].invalid)
6054                         break;
6055
6056                 if (instr[i].type != INSTR_HDR_EXTRACT)
6057                         break;
6058
6059                 if (i == RTE_DIM(instr->io.hdr.header_id))
6060                         break;
6061
6062                 if (i && data[i].n_users)
6063                         break;
6064         }
6065
6066         if (i < 2)
6067                 return 0;
6068
6069         *n_pattern_instr = i;
6070         return 1;
6071 }
6072
6073 static void
6074 instr_pattern_extract_many_replace(struct instruction *instr,
6075                                    struct instruction_data *data,
6076                                    uint32_t n_instr)
6077 {
6078         uint32_t i;
6079
6080         for (i = 1; i < n_instr; i++) {
6081                 instr[0].type++;
6082                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6083                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6084                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6085
6086                 data[i].invalid = 1;
6087         }
6088 }
6089
6090 static uint32_t
6091 instr_pattern_extract_many_optimize(struct instruction *instructions,
6092                                     struct instruction_data *instruction_data,
6093                                     uint32_t n_instructions)
6094 {
6095         uint32_t i;
6096
6097         for (i = 0; i < n_instructions; ) {
6098                 struct instruction *instr = &instructions[i];
6099                 struct instruction_data *data = &instruction_data[i];
6100                 uint32_t n_instr = 0;
6101                 int detected;
6102
6103                 /* Extract many. */
6104                 detected = instr_pattern_extract_many_search(instr,
6105                                                              data,
6106                                                              n_instructions - i,
6107                                                              &n_instr);
6108                 if (detected) {
6109                         instr_pattern_extract_many_replace(instr,
6110                                                            data,
6111                                                            n_instr);
6112                         i += n_instr;
6113                         continue;
6114                 }
6115
6116                 /* No pattern starting at the current instruction. */
6117                 i++;
6118         }
6119
6120         /* Eliminate the invalid instructions that have been optimized out. */
6121         n_instructions = instr_compact(instructions,
6122                                        instruction_data,
6123                                        n_instructions);
6124
6125         return n_instructions;
6126 }
6127
6128 static int
6129 instr_pattern_emit_many_tx_search(struct instruction *instr,
6130                                   struct instruction_data *data,
6131                                   uint32_t n_instr,
6132                                   uint32_t *n_pattern_instr)
6133 {
6134         uint32_t i;
6135
6136         for (i = 0; i < n_instr; i++) {
6137                 if (data[i].invalid)
6138                         break;
6139
6140                 if (instr[i].type != INSTR_HDR_EMIT)
6141                         break;
6142
6143                 if (i == RTE_DIM(instr->io.hdr.header_id))
6144                         break;
6145
6146                 if (i && data[i].n_users)
6147                         break;
6148         }
6149
6150         if (!i)
6151                 return 0;
6152
6153         if (!instruction_is_tx(instr[i].type))
6154                 return 0;
6155
6156         if (data[i].n_users)
6157                 return 0;
6158
6159         i++;
6160
6161         *n_pattern_instr = i;
6162         return 1;
6163 }
6164
6165 static void
6166 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6167                                    struct instruction_data *data,
6168                                    uint32_t n_instr)
6169 {
6170         uint32_t i;
6171
6172         /* Any emit instruction in addition to the first one. */
6173         for (i = 1; i < n_instr - 1; i++) {
6174                 instr[0].type++;
6175                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6176                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6177                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6178
6179                 data[i].invalid = 1;
6180         }
6181
6182         /* The TX instruction is the last one in the pattern. */
6183         instr[0].type++;
6184         instr[0].io.io.offset = instr[i].io.io.offset;
6185         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6186         data[i].invalid = 1;
6187 }
6188
6189 static uint32_t
6190 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6191                                     struct instruction_data *instruction_data,
6192                                     uint32_t n_instructions)
6193 {
6194         uint32_t i;
6195
6196         for (i = 0; i < n_instructions; ) {
6197                 struct instruction *instr = &instructions[i];
6198                 struct instruction_data *data = &instruction_data[i];
6199                 uint32_t n_instr = 0;
6200                 int detected;
6201
6202                 /* Emit many + TX. */
6203                 detected = instr_pattern_emit_many_tx_search(instr,
6204                                                              data,
6205                                                              n_instructions - i,
6206                                                              &n_instr);
6207                 if (detected) {
6208                         instr_pattern_emit_many_tx_replace(instr,
6209                                                            data,
6210                                                            n_instr);
6211                         i += n_instr;
6212                         continue;
6213                 }
6214
6215                 /* No pattern starting at the current instruction. */
6216                 i++;
6217         }
6218
6219         /* Eliminate the invalid instructions that have been optimized out. */
6220         n_instructions = instr_compact(instructions,
6221                                        instruction_data,
6222                                        n_instructions);
6223
6224         return n_instructions;
6225 }
6226
6227 static uint32_t
6228 action_arg_src_mov_count(struct action *a,
6229                          uint32_t arg_id,
6230                          struct instruction *instructions,
6231                          struct instruction_data *instruction_data,
6232                          uint32_t n_instructions);
6233
6234 static int
6235 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6236                                       struct action *a,
6237                                       struct instruction *instr,
6238                                       struct instruction_data *data,
6239                                       uint32_t n_instr,
6240                                       struct instruction *instructions,
6241                                       struct instruction_data *instruction_data,
6242                                       uint32_t n_instructions,
6243                                       uint32_t *n_pattern_instr)
6244 {
6245         struct header *h;
6246         uint32_t src_field_id, i, j;
6247
6248         /* Prerequisites. */
6249         if (!a || !a->st)
6250                 return 0;
6251
6252         /* First instruction: MOV_HM. */
6253         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6254                 return 0;
6255
6256         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6257         if (!h || h->st->var_size)
6258                 return 0;
6259
6260         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6261                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6262                         break;
6263
6264         if (src_field_id == a->st->n_fields)
6265                 return 0;
6266
6267         if (instr[0].mov.dst.offset ||
6268             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6269             instr[0].mov.src.struct_id ||
6270             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6271             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6272                 return 0;
6273
6274         if ((n_instr < h->st->n_fields + 1) ||
6275              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6276                 return 0;
6277
6278         /* Subsequent instructions: MOV_HM. */
6279         for (i = 1; i < h->st->n_fields; i++)
6280                 if (data[i].invalid ||
6281                     data[i].n_users ||
6282                     (instr[i].type != INSTR_MOV_HM) ||
6283                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6284                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6285                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6286                     instr[i].mov.src.struct_id ||
6287                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6288                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6289                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6290                         return 0;
6291
6292         /* Last instruction: HDR_VALIDATE. */
6293         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6294             (instr[i].valid.header_id != h->id))
6295                 return 0;
6296
6297         /* Check that none of the action args that are used as source for this
6298          * DMA transfer are not used as source in any other mov instruction.
6299          */
6300         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6301                 uint32_t n_users;
6302
6303                 n_users = action_arg_src_mov_count(a,
6304                                                    j,
6305                                                    instructions,
6306                                                    instruction_data,
6307                                                    n_instructions);
6308                 if (n_users > 1)
6309                         return 0;
6310         }
6311
6312         *n_pattern_instr = 1 + i;
6313         return 1;
6314 }
6315
6316 static void
6317 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6318                                        struct action *a,
6319                                        struct instruction *instr,
6320                                        struct instruction_data *data,
6321                                        uint32_t n_instr)
6322 {
6323         struct header *h;
6324         uint32_t src_field_id, src_offset, i;
6325
6326         /* Read from the instructions before they are modified. */
6327         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6328         if (!h)
6329                 return;
6330
6331         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6332                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6333                         break;
6334
6335         if (src_field_id == a->st->n_fields)
6336                 return;
6337
6338         src_offset = instr[0].mov.src.offset;
6339
6340         /* Modify the instructions. */
6341         instr[0].type = INSTR_DMA_HT;
6342         instr[0].dma.dst.header_id[0] = h->id;
6343         instr[0].dma.dst.struct_id[0] = h->struct_id;
6344         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6345         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6346
6347         for (i = 1; i < n_instr; i++)
6348                 data[i].invalid = 1;
6349
6350         /* Update the endianness of the action arguments to header endianness. */
6351         for (i = 0; i < h->st->n_fields; i++)
6352                 a->args_endianness[src_field_id + i] = 1;
6353 }
6354
6355 static uint32_t
6356 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6357                                         struct action *a,
6358                                         struct instruction *instructions,
6359                                         struct instruction_data *instruction_data,
6360                                         uint32_t n_instructions)
6361 {
6362         uint32_t i;
6363
6364         if (!a || !a->st)
6365                 return n_instructions;
6366
6367         for (i = 0; i < n_instructions; ) {
6368                 struct instruction *instr = &instructions[i];
6369                 struct instruction_data *data = &instruction_data[i];
6370                 uint32_t n_instr = 0;
6371                 int detected;
6372
6373                 /* Mov all + validate. */
6374                 detected = instr_pattern_mov_all_validate_search(p,
6375                                                                  a,
6376                                                                  instr,
6377                                                                  data,
6378                                                                  n_instructions - i,
6379                                                                  instructions,
6380                                                                  instruction_data,
6381                                                                  n_instructions,
6382                                                                  &n_instr);
6383                 if (detected) {
6384                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6385                         i += n_instr;
6386                         continue;
6387                 }
6388
6389                 /* No pattern starting at the current instruction. */
6390                 i++;
6391         }
6392
6393         /* Eliminate the invalid instructions that have been optimized out. */
6394         n_instructions = instr_compact(instructions,
6395                                        instruction_data,
6396                                        n_instructions);
6397
6398         return n_instructions;
6399 }
6400
6401 static int
6402 instr_pattern_dma_many_search(struct instruction *instr,
6403                               struct instruction_data *data,
6404                               uint32_t n_instr,
6405                               uint32_t *n_pattern_instr)
6406 {
6407         uint32_t i;
6408
6409         for (i = 0; i < n_instr; i++) {
6410                 if (data[i].invalid)
6411                         break;
6412
6413                 if (instr[i].type != INSTR_DMA_HT)
6414                         break;
6415
6416                 if (i == RTE_DIM(instr->dma.dst.header_id))
6417                         break;
6418
6419                 if (i && data[i].n_users)
6420                         break;
6421         }
6422
6423         if (i < 2)
6424                 return 0;
6425
6426         *n_pattern_instr = i;
6427         return 1;
6428 }
6429
6430 static void
6431 instr_pattern_dma_many_replace(struct instruction *instr,
6432                                struct instruction_data *data,
6433                                uint32_t n_instr)
6434 {
6435         uint32_t i;
6436
6437         for (i = 1; i < n_instr; i++) {
6438                 instr[0].type++;
6439                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6440                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6441                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6442                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6443
6444                 data[i].invalid = 1;
6445         }
6446 }
6447
6448 static uint32_t
6449 instr_pattern_dma_many_optimize(struct instruction *instructions,
6450                struct instruction_data *instruction_data,
6451                uint32_t n_instructions)
6452 {
6453         uint32_t i;
6454
6455         for (i = 0; i < n_instructions; ) {
6456                 struct instruction *instr = &instructions[i];
6457                 struct instruction_data *data = &instruction_data[i];
6458                 uint32_t n_instr = 0;
6459                 int detected;
6460
6461                 /* DMA many. */
6462                 detected = instr_pattern_dma_many_search(instr,
6463                                                          data,
6464                                                          n_instructions - i,
6465                                                          &n_instr);
6466                 if (detected) {
6467                         instr_pattern_dma_many_replace(instr, data, n_instr);
6468                         i += n_instr;
6469                         continue;
6470                 }
6471
6472                 /* No pattern starting at the current instruction. */
6473                 i++;
6474         }
6475
6476         /* Eliminate the invalid instructions that have been optimized out. */
6477         n_instructions = instr_compact(instructions,
6478                                        instruction_data,
6479                                        n_instructions);
6480
6481         return n_instructions;
6482 }
6483
6484 static uint32_t
6485 instr_optimize(struct rte_swx_pipeline *p,
6486                struct action *a,
6487                struct instruction *instructions,
6488                struct instruction_data *instruction_data,
6489                uint32_t n_instructions)
6490 {
6491         /* Extract many. */
6492         n_instructions = instr_pattern_extract_many_optimize(instructions,
6493                                                              instruction_data,
6494                                                              n_instructions);
6495
6496         /* Emit many + TX. */
6497         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6498                                                              instruction_data,
6499                                                              n_instructions);
6500
6501         /* Mov all + validate. */
6502         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6503                                                                  a,
6504                                                                  instructions,
6505                                                                  instruction_data,
6506                                                                  n_instructions);
6507
6508         /* DMA many. */
6509         n_instructions = instr_pattern_dma_many_optimize(instructions,
6510                                                          instruction_data,
6511                                                          n_instructions);
6512
6513         return n_instructions;
6514 }
6515
6516 static int
6517 instruction_config(struct rte_swx_pipeline *p,
6518                    struct action *a,
6519                    const char **instructions,
6520                    uint32_t n_instructions)
6521 {
6522         struct instruction *instr = NULL;
6523         struct instruction_data *data = NULL;
6524         int err = 0;
6525         uint32_t i;
6526
6527         CHECK(n_instructions, EINVAL);
6528         CHECK(instructions, EINVAL);
6529         for (i = 0; i < n_instructions; i++)
6530                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6531
6532         /* Memory allocation. */
6533         instr = calloc(n_instructions, sizeof(struct instruction));
6534         if (!instr) {
6535                 err = -ENOMEM;
6536                 goto error;
6537         }
6538
6539         data = calloc(n_instructions, sizeof(struct instruction_data));
6540         if (!data) {
6541                 err = -ENOMEM;
6542                 goto error;
6543         }
6544
6545         for (i = 0; i < n_instructions; i++) {
6546                 char *string = strdup(instructions[i]);
6547                 if (!string) {
6548                         err = -ENOMEM;
6549                         goto error;
6550                 }
6551
6552                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6553                 if (err) {
6554                         free(string);
6555                         goto error;
6556                 }
6557
6558                 free(string);
6559         }
6560
6561         err = instr_label_check(data, n_instructions);
6562         if (err)
6563                 goto error;
6564
6565         err = instr_verify(p, a, instr, data, n_instructions);
6566         if (err)
6567                 goto error;
6568
6569         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6570
6571         err = instr_jmp_resolve(instr, data, n_instructions);
6572         if (err)
6573                 goto error;
6574
6575         if (a) {
6576                 a->instructions = instr;
6577                 a->instruction_data = data;
6578                 a->n_instructions = n_instructions;
6579         } else {
6580                 p->instructions = instr;
6581                 p->instruction_data = data;
6582                 p->n_instructions = n_instructions;
6583         }
6584
6585         return 0;
6586
6587 error:
6588         free(data);
6589         free(instr);
6590         return err;
6591 }
6592
6593 static instr_exec_t instruction_table[] = {
6594         [INSTR_RX] = instr_rx_exec,
6595         [INSTR_TX] = instr_tx_exec,
6596         [INSTR_TX_I] = instr_tx_i_exec,
6597
6598         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6599         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6600         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6601         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6602         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6603         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6604         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6605         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6606         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6607         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6608
6609         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6610         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6611         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6612         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6613         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6614         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6615         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6616         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6617         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6618
6619         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6620         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6621
6622         [INSTR_MOV] = instr_mov_exec,
6623         [INSTR_MOV_MH] = instr_mov_mh_exec,
6624         [INSTR_MOV_HM] = instr_mov_hm_exec,
6625         [INSTR_MOV_HH] = instr_mov_hh_exec,
6626         [INSTR_MOV_I] = instr_mov_i_exec,
6627
6628         [INSTR_DMA_HT] = instr_dma_ht_exec,
6629         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6630         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6631         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6632         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6633         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6634         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6635         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6636
6637         [INSTR_ALU_ADD] = instr_alu_add_exec,
6638         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6639         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6640         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6641         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6642         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6643
6644         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6645         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6646         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6647         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6648         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6649         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6650
6651         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6652         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6653         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6654         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6655
6656         [INSTR_ALU_AND] = instr_alu_and_exec,
6657         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6658         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6659         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6660         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6661
6662         [INSTR_ALU_OR] = instr_alu_or_exec,
6663         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6664         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6665         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6666         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6667
6668         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6669         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6670         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6671         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6672         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6673
6674         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6675         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6676         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6677         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6678         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6679         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6680
6681         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6682         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6683         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6684         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6685         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6686         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6687
6688         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6689         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6690         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6691
6692         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6693         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6694         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6695         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6696         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6697         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6698
6699         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6700         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6701         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6702         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6703         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6704         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6705         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6706         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6707         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6708
6709         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6710         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6711         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6712         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6713         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6714         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6715         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6716         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6717         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6718
6719         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6720         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6721         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6722
6723         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6724         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6725         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6726         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6727         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6728         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6729         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6730         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6731         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6732         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6733         [INSTR_METER_IMM] = instr_meter_imm_exec,
6734         [INSTR_METER_IMI] = instr_meter_imi_exec,
6735
6736         [INSTR_TABLE] = instr_table_exec,
6737         [INSTR_TABLE_AF] = instr_table_af_exec,
6738         [INSTR_SELECTOR] = instr_selector_exec,
6739         [INSTR_LEARNER] = instr_learner_exec,
6740         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6741         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6742         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6743         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6744         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6745
6746         [INSTR_JMP] = instr_jmp_exec,
6747         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6748         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6749         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6750         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6751         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6752         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6753
6754         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6755         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6756         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6757         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6758         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6759
6760         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6761         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6762         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6763         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6764         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6765
6766         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6767         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6768         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6769         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6770         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6771         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6772
6773         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6774         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6775         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6776         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6777         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6778         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6779
6780         [INSTR_RETURN] = instr_return_exec,
6781 };
6782
6783 static int
6784 instruction_table_build(struct rte_swx_pipeline *p)
6785 {
6786         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6787                                       sizeof(struct instr_exec_t *));
6788         if (!p->instruction_table)
6789                 return -EINVAL;
6790
6791         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6792
6793         return 0;
6794 }
6795
6796 static void
6797 instruction_table_build_free(struct rte_swx_pipeline *p)
6798 {
6799         if (!p->instruction_table)
6800                 return;
6801
6802         free(p->instruction_table);
6803         p->instruction_table = NULL;
6804 }
6805
6806 static void
6807 instruction_table_free(struct rte_swx_pipeline *p)
6808 {
6809         instruction_table_build_free(p);
6810 }
6811
6812 static inline void
6813 instr_exec(struct rte_swx_pipeline *p)
6814 {
6815         struct thread *t = &p->threads[p->thread_id];
6816         struct instruction *ip = t->ip;
6817         instr_exec_t instr = p->instruction_table[ip->type];
6818
6819         instr(p);
6820 }
6821
6822 /*
6823  * Action.
6824  */
6825 static struct action *
6826 action_find(struct rte_swx_pipeline *p, const char *name)
6827 {
6828         struct action *elem;
6829
6830         if (!name)
6831                 return NULL;
6832
6833         TAILQ_FOREACH(elem, &p->actions, node)
6834                 if (strcmp(elem->name, name) == 0)
6835                         return elem;
6836
6837         return NULL;
6838 }
6839
6840 static struct action *
6841 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6842 {
6843         struct action *action = NULL;
6844
6845         TAILQ_FOREACH(action, &p->actions, node)
6846                 if (action->id == id)
6847                         return action;
6848
6849         return NULL;
6850 }
6851
6852 static struct field *
6853 action_field_find(struct action *a, const char *name)
6854 {
6855         return a->st ? struct_type_field_find(a->st, name) : NULL;
6856 }
6857
6858 static struct field *
6859 action_field_parse(struct action *action, const char *name)
6860 {
6861         if (name[0] != 't' || name[1] != '.')
6862                 return NULL;
6863
6864         return action_field_find(action, &name[2]);
6865 }
6866
6867 static int
6868 action_has_nbo_args(struct action *a)
6869 {
6870         uint32_t i;
6871
6872         /* Return if the action does not have any args. */
6873         if (!a->st)
6874                 return 0; /* FALSE */
6875
6876         for (i = 0; i < a->st->n_fields; i++)
6877                 if (a->args_endianness[i])
6878                         return 1; /* TRUE */
6879
6880         return 0; /* FALSE */
6881 }
6882
6883 static int
6884 action_does_learning(struct action *a)
6885 {
6886         uint32_t i;
6887
6888         for (i = 0; i < a->n_instructions; i++)
6889                 switch (a->instructions[i].type) {
6890                 case INSTR_LEARNER_LEARN:
6891                         return 1; /* TRUE */
6892
6893                 case INSTR_LEARNER_FORGET:
6894                         return 1; /* TRUE */
6895
6896                 default:
6897                         continue;
6898                 }
6899
6900         return 0; /* FALSE */
6901 }
6902
6903 int
6904 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6905                                const char *name,
6906                                const char *args_struct_type_name,
6907                                const char **instructions,
6908                                uint32_t n_instructions)
6909 {
6910         struct struct_type *args_struct_type = NULL;
6911         struct action *a;
6912         int err;
6913
6914         CHECK(p, EINVAL);
6915
6916         CHECK_NAME(name, EINVAL);
6917         CHECK(!action_find(p, name), EEXIST);
6918
6919         if (args_struct_type_name) {
6920                 CHECK_NAME(args_struct_type_name, EINVAL);
6921                 args_struct_type = struct_type_find(p, args_struct_type_name);
6922                 CHECK(args_struct_type, EINVAL);
6923                 CHECK(!args_struct_type->var_size, EINVAL);
6924         }
6925
6926         /* Node allocation. */
6927         a = calloc(1, sizeof(struct action));
6928         CHECK(a, ENOMEM);
6929         if (args_struct_type) {
6930                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6931                 if (!a->args_endianness) {
6932                         free(a);
6933                         CHECK(0, ENOMEM);
6934                 }
6935         }
6936
6937         /* Node initialization. */
6938         strcpy(a->name, name);
6939         a->st = args_struct_type;
6940         a->id = p->n_actions;
6941
6942         /* Instruction translation. */
6943         err = instruction_config(p, a, instructions, n_instructions);
6944         if (err) {
6945                 free(a->args_endianness);
6946                 free(a);
6947                 return err;
6948         }
6949
6950         /* Node add to tailq. */
6951         TAILQ_INSERT_TAIL(&p->actions, a, node);
6952         p->n_actions++;
6953
6954         return 0;
6955 }
6956
6957 static int
6958 action_build(struct rte_swx_pipeline *p)
6959 {
6960         struct action *action;
6961
6962         /* p->action_instructions. */
6963         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
6964         CHECK(p->action_instructions, ENOMEM);
6965
6966         TAILQ_FOREACH(action, &p->actions, node)
6967                 p->action_instructions[action->id] = action->instructions;
6968
6969         /* p->action_funcs. */
6970         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
6971         CHECK(p->action_funcs, ENOMEM);
6972
6973         return 0;
6974 }
6975
6976 static void
6977 action_build_free(struct rte_swx_pipeline *p)
6978 {
6979         free(p->action_funcs);
6980         p->action_funcs = NULL;
6981
6982         free(p->action_instructions);
6983         p->action_instructions = NULL;
6984 }
6985
6986 static void
6987 action_free(struct rte_swx_pipeline *p)
6988 {
6989         action_build_free(p);
6990
6991         for ( ; ; ) {
6992                 struct action *action;
6993
6994                 action = TAILQ_FIRST(&p->actions);
6995                 if (!action)
6996                         break;
6997
6998                 TAILQ_REMOVE(&p->actions, action, node);
6999                 free(action->instruction_data);
7000                 free(action->instructions);
7001                 free(action);
7002         }
7003 }
7004
7005 static uint32_t
7006 action_arg_src_mov_count(struct action *a,
7007                          uint32_t arg_id,
7008                          struct instruction *instructions,
7009                          struct instruction_data *instruction_data,
7010                          uint32_t n_instructions)
7011 {
7012         uint32_t offset, n_users = 0, i;
7013
7014         if (!a->st ||
7015             (arg_id >= a->st->n_fields) ||
7016             !instructions ||
7017             !instruction_data ||
7018             !n_instructions)
7019                 return 0;
7020
7021         offset = a->st->fields[arg_id].offset / 8;
7022
7023         for (i = 0; i < n_instructions; i++) {
7024                 struct instruction *instr = &instructions[i];
7025                 struct instruction_data *data = &instruction_data[i];
7026
7027                 if (data->invalid ||
7028                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7029                     instr->mov.src.struct_id ||
7030                     (instr->mov.src.offset != offset))
7031                         continue;
7032
7033                 n_users++;
7034         }
7035
7036         return n_users;
7037 }
7038
7039 /*
7040  * Table.
7041  */
7042 static struct table_type *
7043 table_type_find(struct rte_swx_pipeline *p, const char *name)
7044 {
7045         struct table_type *elem;
7046
7047         TAILQ_FOREACH(elem, &p->table_types, node)
7048                 if (strcmp(elem->name, name) == 0)
7049                         return elem;
7050
7051         return NULL;
7052 }
7053
7054 static struct table_type *
7055 table_type_resolve(struct rte_swx_pipeline *p,
7056                    const char *recommended_type_name,
7057                    enum rte_swx_table_match_type match_type)
7058 {
7059         struct table_type *elem;
7060
7061         /* Only consider the recommended type if the match type is correct. */
7062         if (recommended_type_name)
7063                 TAILQ_FOREACH(elem, &p->table_types, node)
7064                         if (!strcmp(elem->name, recommended_type_name) &&
7065                             (elem->match_type == match_type))
7066                                 return elem;
7067
7068         /* Ignore the recommended type and get the first element with this match
7069          * type.
7070          */
7071         TAILQ_FOREACH(elem, &p->table_types, node)
7072                 if (elem->match_type == match_type)
7073                         return elem;
7074
7075         return NULL;
7076 }
7077
7078 static struct table *
7079 table_find(struct rte_swx_pipeline *p, const char *name)
7080 {
7081         struct table *elem;
7082
7083         TAILQ_FOREACH(elem, &p->tables, node)
7084                 if (strcmp(elem->name, name) == 0)
7085                         return elem;
7086
7087         return NULL;
7088 }
7089
7090 static struct table *
7091 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7092 {
7093         struct table *table = NULL;
7094
7095         TAILQ_FOREACH(table, &p->tables, node)
7096                 if (table->id == id)
7097                         return table;
7098
7099         return NULL;
7100 }
7101
7102 int
7103 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7104                                      const char *name,
7105                                      enum rte_swx_table_match_type match_type,
7106                                      struct rte_swx_table_ops *ops)
7107 {
7108         struct table_type *elem;
7109
7110         CHECK(p, EINVAL);
7111
7112         CHECK_NAME(name, EINVAL);
7113         CHECK(!table_type_find(p, name), EEXIST);
7114
7115         CHECK(ops, EINVAL);
7116         CHECK(ops->create, EINVAL);
7117         CHECK(ops->lkp, EINVAL);
7118         CHECK(ops->free, EINVAL);
7119
7120         /* Node allocation. */
7121         elem = calloc(1, sizeof(struct table_type));
7122         CHECK(elem, ENOMEM);
7123
7124         /* Node initialization. */
7125         strcpy(elem->name, name);
7126         elem->match_type = match_type;
7127         memcpy(&elem->ops, ops, sizeof(*ops));
7128
7129         /* Node add to tailq. */
7130         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7131
7132         return 0;
7133 }
7134
7135 static int
7136 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7137                          uint32_t n_fields,
7138                          enum rte_swx_table_match_type *match_type)
7139 {
7140         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7141
7142         for (i = 0; i < n_fields; i++) {
7143                 struct rte_swx_match_field_params  *f = &fields[i];
7144
7145                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7146                         n_fields_em++;
7147
7148                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7149                         n_fields_lpm++;
7150         }
7151
7152         if ((n_fields_lpm > 1) ||
7153             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7154                 return -EINVAL;
7155
7156         *match_type = (n_fields_em == n_fields) ?
7157                        RTE_SWX_TABLE_MATCH_EXACT :
7158                        RTE_SWX_TABLE_MATCH_WILDCARD;
7159
7160         return 0;
7161 }
7162
7163 static int
7164 table_match_fields_check(struct rte_swx_pipeline *p,
7165                          struct rte_swx_pipeline_table_params *params,
7166                          struct header **header)
7167 {
7168         struct header *h0 = NULL;
7169         struct field *hf, *mf;
7170         uint32_t *offset = NULL, i;
7171         int status = 0;
7172
7173         /* Return if no match fields. */
7174         if (!params->n_fields) {
7175                 if (params->fields) {
7176                         status = -EINVAL;
7177                         goto end;
7178                 }
7179
7180                 if (header)
7181                         *header = NULL;
7182
7183                 return 0;
7184         }
7185
7186         /* Memory allocation. */
7187         offset = calloc(params->n_fields, sizeof(uint32_t));
7188         if (!offset) {
7189                 status = -ENOMEM;
7190                 goto end;
7191         }
7192
7193         /* Check that all the match fields belong to either the same header or
7194          * to the meta-data.
7195          */
7196         hf = header_field_parse(p, params->fields[0].name, &h0);
7197         mf = metadata_field_parse(p, params->fields[0].name);
7198         if ((!hf && !mf) || (hf && hf->var_size)) {
7199                 status = -EINVAL;
7200                 goto end;
7201         }
7202
7203         offset[0] = h0 ? hf->offset : mf->offset;
7204
7205         for (i = 1; i < params->n_fields; i++)
7206                 if (h0) {
7207                         struct header *h;
7208
7209                         hf = header_field_parse(p, params->fields[i].name, &h);
7210                         if (!hf || (h->id != h0->id) || hf->var_size) {
7211                                 status = -EINVAL;
7212                                 goto end;
7213                         }
7214
7215                         offset[i] = hf->offset;
7216                 } else {
7217                         mf = metadata_field_parse(p, params->fields[i].name);
7218                         if (!mf) {
7219                                 status = -EINVAL;
7220                                 goto end;
7221                         }
7222
7223                         offset[i] = mf->offset;
7224                 }
7225
7226         /* Check that there are no duplicated match fields. */
7227         for (i = 0; i < params->n_fields; i++) {
7228                 uint32_t j;
7229
7230                 for (j = 0; j < i; j++)
7231                         if (offset[j] == offset[i]) {
7232                                 status = -EINVAL;
7233                                 goto end;
7234                         }
7235         }
7236
7237         /* Return. */
7238         if (header)
7239                 *header = h0;
7240
7241 end:
7242         free(offset);
7243         return status;
7244 }
7245
7246 int
7247 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7248                               const char *name,
7249                               struct rte_swx_pipeline_table_params *params,
7250                               const char *recommended_table_type_name,
7251                               const char *args,
7252                               uint32_t size)
7253 {
7254         struct table_type *type;
7255         struct table *t;
7256         struct action *default_action;
7257         struct header *header = NULL;
7258         uint32_t action_data_size_max = 0, i;
7259         int status = 0;
7260
7261         CHECK(p, EINVAL);
7262
7263         CHECK_NAME(name, EINVAL);
7264         CHECK(!table_find(p, name), EEXIST);
7265         CHECK(!selector_find(p, name), EEXIST);
7266         CHECK(!learner_find(p, name), EEXIST);
7267
7268         CHECK(params, EINVAL);
7269
7270         /* Match checks. */
7271         status = table_match_fields_check(p, params, &header);
7272         if (status)
7273                 return status;
7274
7275         /* Action checks. */
7276         CHECK(params->n_actions, EINVAL);
7277         CHECK(params->action_names, EINVAL);
7278         for (i = 0; i < params->n_actions; i++) {
7279                 const char *action_name = params->action_names[i];
7280                 struct action *a;
7281                 uint32_t action_data_size;
7282
7283                 CHECK_NAME(action_name, EINVAL);
7284
7285                 a = action_find(p, action_name);
7286                 CHECK(a, EINVAL);
7287                 CHECK(!action_does_learning(a), EINVAL);
7288
7289                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7290                 if (action_data_size > action_data_size_max)
7291                         action_data_size_max = action_data_size;
7292         }
7293
7294         CHECK_NAME(params->default_action_name, EINVAL);
7295         for (i = 0; i < p->n_actions; i++)
7296                 if (!strcmp(params->action_names[i],
7297                             params->default_action_name))
7298                         break;
7299         CHECK(i < params->n_actions, EINVAL);
7300         default_action = action_find(p, params->default_action_name);
7301         CHECK((default_action->st && params->default_action_data) ||
7302               !params->default_action_data, EINVAL);
7303
7304         /* Table type checks. */
7305         if (recommended_table_type_name)
7306                 CHECK_NAME(recommended_table_type_name, EINVAL);
7307
7308         if (params->n_fields) {
7309                 enum rte_swx_table_match_type match_type;
7310
7311                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7312                 if (status)
7313                         return status;
7314
7315                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7316                 CHECK(type, EINVAL);
7317         } else {
7318                 type = NULL;
7319         }
7320
7321         /* Memory allocation. */
7322         t = calloc(1, sizeof(struct table));
7323         CHECK(t, ENOMEM);
7324
7325         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7326         if (!t->fields) {
7327                 free(t);
7328                 CHECK(0, ENOMEM);
7329         }
7330
7331         t->actions = calloc(params->n_actions, sizeof(struct action *));
7332         if (!t->actions) {
7333                 free(t->fields);
7334                 free(t);
7335                 CHECK(0, ENOMEM);
7336         }
7337
7338         if (action_data_size_max) {
7339                 t->default_action_data = calloc(1, action_data_size_max);
7340                 if (!t->default_action_data) {
7341                         free(t->actions);
7342                         free(t->fields);
7343                         free(t);
7344                         CHECK(0, ENOMEM);
7345                 }
7346         }
7347
7348         /* Node initialization. */
7349         strcpy(t->name, name);
7350         if (args && args[0])
7351                 strcpy(t->args, args);
7352         t->type = type;
7353
7354         for (i = 0; i < params->n_fields; i++) {
7355                 struct rte_swx_match_field_params *field = &params->fields[i];
7356                 struct match_field *f = &t->fields[i];
7357
7358                 f->match_type = field->match_type;
7359                 f->field = header ?
7360                         header_field_parse(p, field->name, NULL) :
7361                         metadata_field_parse(p, field->name);
7362         }
7363         t->n_fields = params->n_fields;
7364         t->header = header;
7365
7366         for (i = 0; i < params->n_actions; i++)
7367                 t->actions[i] = action_find(p, params->action_names[i]);
7368         t->default_action = default_action;
7369         if (default_action->st)
7370                 memcpy(t->default_action_data,
7371                        params->default_action_data,
7372                        default_action->st->n_bits / 8);
7373         t->n_actions = params->n_actions;
7374         t->default_action_is_const = params->default_action_is_const;
7375         t->action_data_size_max = action_data_size_max;
7376
7377         t->size = size;
7378         t->id = p->n_tables;
7379
7380         /* Node add to tailq. */
7381         TAILQ_INSERT_TAIL(&p->tables, t, node);
7382         p->n_tables++;
7383
7384         return 0;
7385 }
7386
7387 static struct rte_swx_table_params *
7388 table_params_get(struct table *table)
7389 {
7390         struct rte_swx_table_params *params;
7391         struct field *first, *last;
7392         uint8_t *key_mask;
7393         uint32_t key_size, key_offset, action_data_size, i;
7394
7395         /* Memory allocation. */
7396         params = calloc(1, sizeof(struct rte_swx_table_params));
7397         if (!params)
7398                 return NULL;
7399
7400         /* Find first (smallest offset) and last (biggest offset) match fields. */
7401         first = table->fields[0].field;
7402         last = table->fields[0].field;
7403
7404         for (i = 0; i < table->n_fields; i++) {
7405                 struct field *f = table->fields[i].field;
7406
7407                 if (f->offset < first->offset)
7408                         first = f;
7409
7410                 if (f->offset > last->offset)
7411                         last = f;
7412         }
7413
7414         /* Key offset and size. */
7415         key_offset = first->offset / 8;
7416         key_size = (last->offset + last->n_bits - first->offset) / 8;
7417
7418         /* Memory allocation. */
7419         key_mask = calloc(1, key_size);
7420         if (!key_mask) {
7421                 free(params);
7422                 return NULL;
7423         }
7424
7425         /* Key mask. */
7426         for (i = 0; i < table->n_fields; i++) {
7427                 struct field *f = table->fields[i].field;
7428                 uint32_t start = (f->offset - first->offset) / 8;
7429                 size_t size = f->n_bits / 8;
7430
7431                 memset(&key_mask[start], 0xFF, size);
7432         }
7433
7434         /* Action data size. */
7435         action_data_size = 0;
7436         for (i = 0; i < table->n_actions; i++) {
7437                 struct action *action = table->actions[i];
7438                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7439
7440                 if (ads > action_data_size)
7441                         action_data_size = ads;
7442         }
7443
7444         /* Fill in. */
7445         params->match_type = table->type->match_type;
7446         params->key_size = key_size;
7447         params->key_offset = key_offset;
7448         params->key_mask0 = key_mask;
7449         params->action_data_size = action_data_size;
7450         params->n_keys_max = table->size;
7451
7452         return params;
7453 }
7454
7455 static void
7456 table_params_free(struct rte_swx_table_params *params)
7457 {
7458         if (!params)
7459                 return;
7460
7461         free(params->key_mask0);
7462         free(params);
7463 }
7464
7465 static int
7466 table_stub_lkp(void *table __rte_unused,
7467                void *mailbox __rte_unused,
7468                uint8_t **key __rte_unused,
7469                uint64_t *action_id __rte_unused,
7470                uint8_t **action_data __rte_unused,
7471                int *hit)
7472 {
7473         *hit = 0;
7474         return 1; /* DONE. */
7475 }
7476
7477 static int
7478 table_build(struct rte_swx_pipeline *p)
7479 {
7480         uint32_t i;
7481
7482         /* Per pipeline: table statistics. */
7483         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7484         CHECK(p->table_stats, ENOMEM);
7485
7486         for (i = 0; i < p->n_tables; i++) {
7487                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7488                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7489         }
7490
7491         /* Per thread: table runt-time. */
7492         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7493                 struct thread *t = &p->threads[i];
7494                 struct table *table;
7495
7496                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7497                 CHECK(t->tables, ENOMEM);
7498
7499                 TAILQ_FOREACH(table, &p->tables, node) {
7500                         struct table_runtime *r = &t->tables[table->id];
7501
7502                         if (table->type) {
7503                                 uint64_t size;
7504
7505                                 size = table->type->ops.mailbox_size_get();
7506
7507                                 /* r->func. */
7508                                 r->func = table->type->ops.lkp;
7509
7510                                 /* r->mailbox. */
7511                                 if (size) {
7512                                         r->mailbox = calloc(1, size);
7513                                         CHECK(r->mailbox, ENOMEM);
7514                                 }
7515
7516                                 /* r->key. */
7517                                 r->key = table->header ?
7518                                         &t->structs[table->header->struct_id] :
7519                                         &t->structs[p->metadata_struct_id];
7520                         } else {
7521                                 r->func = table_stub_lkp;
7522                         }
7523                 }
7524         }
7525
7526         return 0;
7527 }
7528
7529 static void
7530 table_build_free(struct rte_swx_pipeline *p)
7531 {
7532         uint32_t i;
7533
7534         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7535                 struct thread *t = &p->threads[i];
7536                 uint32_t j;
7537
7538                 if (!t->tables)
7539                         continue;
7540
7541                 for (j = 0; j < p->n_tables; j++) {
7542                         struct table_runtime *r = &t->tables[j];
7543
7544                         free(r->mailbox);
7545                 }
7546
7547                 free(t->tables);
7548                 t->tables = NULL;
7549         }
7550
7551         if (p->table_stats) {
7552                 for (i = 0; i < p->n_tables; i++)
7553                         free(p->table_stats[i].n_pkts_action);
7554
7555                 free(p->table_stats);
7556         }
7557 }
7558
7559 static void
7560 table_free(struct rte_swx_pipeline *p)
7561 {
7562         table_build_free(p);
7563
7564         /* Tables. */
7565         for ( ; ; ) {
7566                 struct table *elem;
7567
7568                 elem = TAILQ_FIRST(&p->tables);
7569                 if (!elem)
7570                         break;
7571
7572                 TAILQ_REMOVE(&p->tables, elem, node);
7573                 free(elem->fields);
7574                 free(elem->actions);
7575                 free(elem->default_action_data);
7576                 free(elem);
7577         }
7578
7579         /* Table types. */
7580         for ( ; ; ) {
7581                 struct table_type *elem;
7582
7583                 elem = TAILQ_FIRST(&p->table_types);
7584                 if (!elem)
7585                         break;
7586
7587                 TAILQ_REMOVE(&p->table_types, elem, node);
7588                 free(elem);
7589         }
7590 }
7591
7592 /*
7593  * Selector.
7594  */
7595 static struct selector *
7596 selector_find(struct rte_swx_pipeline *p, const char *name)
7597 {
7598         struct selector *s;
7599
7600         TAILQ_FOREACH(s, &p->selectors, node)
7601                 if (strcmp(s->name, name) == 0)
7602                         return s;
7603
7604         return NULL;
7605 }
7606
7607 static struct selector *
7608 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7609 {
7610         struct selector *s = NULL;
7611
7612         TAILQ_FOREACH(s, &p->selectors, node)
7613                 if (s->id == id)
7614                         return s;
7615
7616         return NULL;
7617 }
7618
7619 static int
7620 selector_fields_check(struct rte_swx_pipeline *p,
7621                       struct rte_swx_pipeline_selector_params *params,
7622                       struct header **header)
7623 {
7624         struct header *h0 = NULL;
7625         struct field *hf, *mf;
7626         uint32_t i;
7627
7628         /* Return if no selector fields. */
7629         if (!params->n_selector_fields || !params->selector_field_names)
7630                 return -EINVAL;
7631
7632         /* Check that all the selector fields either belong to the same header
7633          * or are all meta-data fields.
7634          */
7635         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7636         mf = metadata_field_parse(p, params->selector_field_names[0]);
7637         if (!hf && !mf)
7638                 return -EINVAL;
7639
7640         for (i = 1; i < params->n_selector_fields; i++)
7641                 if (h0) {
7642                         struct header *h;
7643
7644                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7645                         if (!hf || (h->id != h0->id))
7646                                 return -EINVAL;
7647                 } else {
7648                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7649                         if (!mf)
7650                                 return -EINVAL;
7651                 }
7652
7653         /* Check that there are no duplicated match fields. */
7654         for (i = 0; i < params->n_selector_fields; i++) {
7655                 const char *field_name = params->selector_field_names[i];
7656                 uint32_t j;
7657
7658                 for (j = i + 1; j < params->n_selector_fields; j++)
7659                         if (!strcmp(params->selector_field_names[j], field_name))
7660                                 return -EINVAL;
7661         }
7662
7663         /* Return. */
7664         if (header)
7665                 *header = h0;
7666
7667         return 0;
7668 }
7669
7670 int
7671 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7672                                  const char *name,
7673                                  struct rte_swx_pipeline_selector_params *params)
7674 {
7675         struct selector *s;
7676         struct header *selector_header = NULL;
7677         struct field *group_id_field, *member_id_field;
7678         uint32_t i;
7679         int status = 0;
7680
7681         CHECK(p, EINVAL);
7682
7683         CHECK_NAME(name, EINVAL);
7684         CHECK(!table_find(p, name), EEXIST);
7685         CHECK(!selector_find(p, name), EEXIST);
7686         CHECK(!learner_find(p, name), EEXIST);
7687
7688         CHECK(params, EINVAL);
7689
7690         CHECK_NAME(params->group_id_field_name, EINVAL);
7691         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7692         CHECK(group_id_field, EINVAL);
7693
7694         for (i = 0; i < params->n_selector_fields; i++) {
7695                 const char *field_name = params->selector_field_names[i];
7696
7697                 CHECK_NAME(field_name, EINVAL);
7698         }
7699         status = selector_fields_check(p, params, &selector_header);
7700         if (status)
7701                 return status;
7702
7703         CHECK_NAME(params->member_id_field_name, EINVAL);
7704         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7705         CHECK(member_id_field, EINVAL);
7706
7707         CHECK(params->n_groups_max, EINVAL);
7708
7709         CHECK(params->n_members_per_group_max, EINVAL);
7710
7711         /* Memory allocation. */
7712         s = calloc(1, sizeof(struct selector));
7713         if (!s) {
7714                 status = -ENOMEM;
7715                 goto error;
7716         }
7717
7718         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7719         if (!s->selector_fields) {
7720                 status = -ENOMEM;
7721                 goto error;
7722         }
7723
7724         /* Node initialization. */
7725         strcpy(s->name, name);
7726
7727         s->group_id_field = group_id_field;
7728
7729         for (i = 0; i < params->n_selector_fields; i++) {
7730                 const char *field_name = params->selector_field_names[i];
7731
7732                 s->selector_fields[i] = selector_header ?
7733                         header_field_parse(p, field_name, NULL) :
7734                         metadata_field_parse(p, field_name);
7735         }
7736
7737         s->n_selector_fields = params->n_selector_fields;
7738
7739         s->selector_header = selector_header;
7740
7741         s->member_id_field = member_id_field;
7742
7743         s->n_groups_max = params->n_groups_max;
7744
7745         s->n_members_per_group_max = params->n_members_per_group_max;
7746
7747         s->id = p->n_selectors;
7748
7749         /* Node add to tailq. */
7750         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7751         p->n_selectors++;
7752
7753         return 0;
7754
7755 error:
7756         if (!s)
7757                 return status;
7758
7759         free(s->selector_fields);
7760
7761         free(s);
7762
7763         return status;
7764 }
7765
7766 static void
7767 selector_params_free(struct rte_swx_table_selector_params *params)
7768 {
7769         if (!params)
7770                 return;
7771
7772         free(params->selector_mask);
7773
7774         free(params);
7775 }
7776
7777 static struct rte_swx_table_selector_params *
7778 selector_table_params_get(struct selector *s)
7779 {
7780         struct rte_swx_table_selector_params *params = NULL;
7781         struct field *first, *last;
7782         uint32_t i;
7783
7784         /* Memory allocation. */
7785         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7786         if (!params)
7787                 goto error;
7788
7789         /* Group ID. */
7790         params->group_id_offset = s->group_id_field->offset / 8;
7791
7792         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7793         first = s->selector_fields[0];
7794         last = s->selector_fields[0];
7795
7796         for (i = 0; i < s->n_selector_fields; i++) {
7797                 struct field *f = s->selector_fields[i];
7798
7799                 if (f->offset < first->offset)
7800                         first = f;
7801
7802                 if (f->offset > last->offset)
7803                         last = f;
7804         }
7805
7806         /* Selector offset and size. */
7807         params->selector_offset = first->offset / 8;
7808         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7809
7810         /* Memory allocation. */
7811         params->selector_mask = calloc(1, params->selector_size);
7812         if (!params->selector_mask)
7813                 goto error;
7814
7815         /* Selector mask. */
7816         for (i = 0; i < s->n_selector_fields; i++) {
7817                 struct field *f = s->selector_fields[i];
7818                 uint32_t start = (f->offset - first->offset) / 8;
7819                 size_t size = f->n_bits / 8;
7820
7821                 memset(&params->selector_mask[start], 0xFF, size);
7822         }
7823
7824         /* Member ID. */
7825         params->member_id_offset = s->member_id_field->offset / 8;
7826
7827         /* Maximum number of groups. */
7828         params->n_groups_max = s->n_groups_max;
7829
7830         /* Maximum number of members per group. */
7831         params->n_members_per_group_max = s->n_members_per_group_max;
7832
7833         return params;
7834
7835 error:
7836         selector_params_free(params);
7837         return NULL;
7838 }
7839
7840 static void
7841 selector_build_free(struct rte_swx_pipeline *p)
7842 {
7843         uint32_t i;
7844
7845         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7846                 struct thread *t = &p->threads[i];
7847                 uint32_t j;
7848
7849                 if (!t->selectors)
7850                         continue;
7851
7852                 for (j = 0; j < p->n_selectors; j++) {
7853                         struct selector_runtime *r = &t->selectors[j];
7854
7855                         free(r->mailbox);
7856                 }
7857
7858                 free(t->selectors);
7859                 t->selectors = NULL;
7860         }
7861
7862         free(p->selector_stats);
7863         p->selector_stats = NULL;
7864 }
7865
7866 static int
7867 selector_build(struct rte_swx_pipeline *p)
7868 {
7869         uint32_t i;
7870         int status = 0;
7871
7872         /* Per pipeline: selector statistics. */
7873         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7874         if (!p->selector_stats) {
7875                 status = -ENOMEM;
7876                 goto error;
7877         }
7878
7879         /* Per thread: selector run-time. */
7880         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7881                 struct thread *t = &p->threads[i];
7882                 struct selector *s;
7883
7884                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7885                 if (!t->selectors) {
7886                         status = -ENOMEM;
7887                         goto error;
7888                 }
7889
7890                 TAILQ_FOREACH(s, &p->selectors, node) {
7891                         struct selector_runtime *r = &t->selectors[s->id];
7892                         uint64_t size;
7893
7894                         /* r->mailbox. */
7895                         size = rte_swx_table_selector_mailbox_size_get();
7896                         if (size) {
7897                                 r->mailbox = calloc(1, size);
7898                                 if (!r->mailbox) {
7899                                         status = -ENOMEM;
7900                                         goto error;
7901                                 }
7902                         }
7903
7904                         /* r->group_id_buffer. */
7905                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7906
7907                         /* r->selector_buffer. */
7908                         r->selector_buffer = s->selector_header ?
7909                                 &t->structs[s->selector_header->struct_id] :
7910                                 &t->structs[p->metadata_struct_id];
7911
7912                         /* r->member_id_buffer. */
7913                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
7914                 }
7915         }
7916
7917         return 0;
7918
7919 error:
7920         selector_build_free(p);
7921         return status;
7922 }
7923
7924 static void
7925 selector_free(struct rte_swx_pipeline *p)
7926 {
7927         selector_build_free(p);
7928
7929         /* Selector tables. */
7930         for ( ; ; ) {
7931                 struct selector *elem;
7932
7933                 elem = TAILQ_FIRST(&p->selectors);
7934                 if (!elem)
7935                         break;
7936
7937                 TAILQ_REMOVE(&p->selectors, elem, node);
7938                 free(elem->selector_fields);
7939                 free(elem);
7940         }
7941 }
7942
7943 /*
7944  * Learner table.
7945  */
7946 static struct learner *
7947 learner_find(struct rte_swx_pipeline *p, const char *name)
7948 {
7949         struct learner *l;
7950
7951         TAILQ_FOREACH(l, &p->learners, node)
7952                 if (!strcmp(l->name, name))
7953                         return l;
7954
7955         return NULL;
7956 }
7957
7958 static struct learner *
7959 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7960 {
7961         struct learner *l = NULL;
7962
7963         TAILQ_FOREACH(l, &p->learners, node)
7964                 if (l->id == id)
7965                         return l;
7966
7967         return NULL;
7968 }
7969
7970 static int
7971 learner_match_fields_check(struct rte_swx_pipeline *p,
7972                            struct rte_swx_pipeline_learner_params *params,
7973                            struct header **header)
7974 {
7975         struct header *h0 = NULL;
7976         struct field *hf, *mf;
7977         uint32_t i;
7978
7979         /* Return if no match fields. */
7980         if (!params->n_fields || !params->field_names)
7981                 return -EINVAL;
7982
7983         /* Check that all the match fields either belong to the same header
7984          * or are all meta-data fields.
7985          */
7986         hf = header_field_parse(p, params->field_names[0], &h0);
7987         mf = metadata_field_parse(p, params->field_names[0]);
7988         if (!hf && !mf)
7989                 return -EINVAL;
7990
7991         for (i = 1; i < params->n_fields; i++)
7992                 if (h0) {
7993                         struct header *h;
7994
7995                         hf = header_field_parse(p, params->field_names[i], &h);
7996                         if (!hf || (h->id != h0->id))
7997                                 return -EINVAL;
7998                 } else {
7999                         mf = metadata_field_parse(p, params->field_names[i]);
8000                         if (!mf)
8001                                 return -EINVAL;
8002                 }
8003
8004         /* Check that there are no duplicated match fields. */
8005         for (i = 0; i < params->n_fields; i++) {
8006                 const char *field_name = params->field_names[i];
8007                 uint32_t j;
8008
8009                 for (j = i + 1; j < params->n_fields; j++)
8010                         if (!strcmp(params->field_names[j], field_name))
8011                                 return -EINVAL;
8012         }
8013
8014         /* Return. */
8015         if (header)
8016                 *header = h0;
8017
8018         return 0;
8019 }
8020
8021 static int
8022 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8023 {
8024         struct struct_type *mst = p->metadata_st, *ast = a->st;
8025         struct field *mf, *af;
8026         uint32_t mf_pos, i;
8027
8028         if (!ast) {
8029                 if (mf_name)
8030                         return -EINVAL;
8031
8032                 return 0;
8033         }
8034
8035         /* Check that mf_name is the name of a valid meta-data field. */
8036         CHECK_NAME(mf_name, EINVAL);
8037         mf = metadata_field_parse(p, mf_name);
8038         CHECK(mf, EINVAL);
8039
8040         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8041          * all the action arguments.
8042          */
8043         mf_pos = mf - mst->fields;
8044         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8045
8046         /* Check that the size of each of the identified meta-data fields matches exactly the size
8047          * of the corresponding action argument.
8048          */
8049         for (i = 0; i < ast->n_fields; i++) {
8050                 mf = &mst->fields[mf_pos + i];
8051                 af = &ast->fields[i];
8052
8053                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8054         }
8055
8056         return 0;
8057 }
8058
8059 static int
8060 learner_action_learning_check(struct rte_swx_pipeline *p,
8061                               struct action *action,
8062                               const char **action_names,
8063                               uint32_t n_actions)
8064 {
8065         uint32_t i;
8066
8067         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8068          * the action passed as argument to the "learn" instruction) is also enabled for the
8069          * current learner table.
8070          */
8071         for (i = 0; i < action->n_instructions; i++) {
8072                 struct instruction *instr = &action->instructions[i];
8073                 uint32_t found = 0, j;
8074
8075                 if (instr->type != INSTR_LEARNER_LEARN)
8076                         continue;
8077
8078                 for (j = 0; j < n_actions; j++) {
8079                         struct action *a;
8080
8081                         a = action_find(p, action_names[j]);
8082                         if (!a)
8083                                 return -EINVAL;
8084
8085                         if (a->id == instr->learn.action_id)
8086                                 found = 1;
8087                 }
8088
8089                 if (!found)
8090                         return -EINVAL;
8091         }
8092
8093         return 0;
8094 }
8095
8096 int
8097 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8098                               const char *name,
8099                               struct rte_swx_pipeline_learner_params *params,
8100                               uint32_t size,
8101                               uint32_t timeout)
8102 {
8103         struct learner *l = NULL;
8104         struct action *default_action;
8105         struct header *header = NULL;
8106         uint32_t action_data_size_max = 0, i;
8107         int status = 0;
8108
8109         CHECK(p, EINVAL);
8110
8111         CHECK_NAME(name, EINVAL);
8112         CHECK(!table_find(p, name), EEXIST);
8113         CHECK(!selector_find(p, name), EEXIST);
8114         CHECK(!learner_find(p, name), EEXIST);
8115
8116         CHECK(params, EINVAL);
8117
8118         /* Match checks. */
8119         status = learner_match_fields_check(p, params, &header);
8120         if (status)
8121                 return status;
8122
8123         /* Action checks. */
8124         CHECK(params->n_actions, EINVAL);
8125
8126         CHECK(params->action_names, EINVAL);
8127         for (i = 0; i < params->n_actions; i++) {
8128                 const char *action_name = params->action_names[i];
8129                 const char *action_field_name = params->action_field_names[i];
8130                 struct action *a;
8131                 uint32_t action_data_size;
8132
8133                 CHECK_NAME(action_name, EINVAL);
8134
8135                 a = action_find(p, action_name);
8136                 CHECK(a, EINVAL);
8137
8138                 status = learner_action_args_check(p, a, action_field_name);
8139                 if (status)
8140                         return status;
8141
8142                 status = learner_action_learning_check(p,
8143                                                        a,
8144                                                        params->action_names,
8145                                                        params->n_actions);
8146                 if (status)
8147                         return status;
8148
8149                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8150                 if (action_data_size > action_data_size_max)
8151                         action_data_size_max = action_data_size;
8152         }
8153
8154         CHECK_NAME(params->default_action_name, EINVAL);
8155         for (i = 0; i < p->n_actions; i++)
8156                 if (!strcmp(params->action_names[i],
8157                             params->default_action_name))
8158                         break;
8159         CHECK(i < params->n_actions, EINVAL);
8160
8161         default_action = action_find(p, params->default_action_name);
8162         CHECK((default_action->st && params->default_action_data) ||
8163               !params->default_action_data, EINVAL);
8164
8165         /* Any other checks. */
8166         CHECK(size, EINVAL);
8167         CHECK(timeout, EINVAL);
8168
8169         /* Memory allocation. */
8170         l = calloc(1, sizeof(struct learner));
8171         if (!l)
8172                 goto nomem;
8173
8174         l->fields = calloc(params->n_fields, sizeof(struct field *));
8175         if (!l->fields)
8176                 goto nomem;
8177
8178         l->actions = calloc(params->n_actions, sizeof(struct action *));
8179         if (!l->actions)
8180                 goto nomem;
8181
8182         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8183         if (!l->action_arg)
8184                 goto nomem;
8185
8186         if (action_data_size_max) {
8187                 l->default_action_data = calloc(1, action_data_size_max);
8188                 if (!l->default_action_data)
8189                         goto nomem;
8190         }
8191
8192         /* Node initialization. */
8193         strcpy(l->name, name);
8194
8195         for (i = 0; i < params->n_fields; i++) {
8196                 const char *field_name = params->field_names[i];
8197
8198                 l->fields[i] = header ?
8199                         header_field_parse(p, field_name, NULL) :
8200                         metadata_field_parse(p, field_name);
8201         }
8202
8203         l->n_fields = params->n_fields;
8204
8205         l->header = header;
8206
8207         for (i = 0; i < params->n_actions; i++) {
8208                 const char *mf_name = params->action_field_names[i];
8209
8210                 l->actions[i] = action_find(p, params->action_names[i]);
8211
8212                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8213         }
8214
8215         l->default_action = default_action;
8216
8217         if (default_action->st)
8218                 memcpy(l->default_action_data,
8219                        params->default_action_data,
8220                        default_action->st->n_bits / 8);
8221
8222         l->n_actions = params->n_actions;
8223
8224         l->default_action_is_const = params->default_action_is_const;
8225
8226         l->action_data_size_max = action_data_size_max;
8227
8228         l->size = size;
8229
8230         l->timeout = timeout;
8231
8232         l->id = p->n_learners;
8233
8234         /* Node add to tailq. */
8235         TAILQ_INSERT_TAIL(&p->learners, l, node);
8236         p->n_learners++;
8237
8238         return 0;
8239
8240 nomem:
8241         if (!l)
8242                 return -ENOMEM;
8243
8244         free(l->action_arg);
8245         free(l->actions);
8246         free(l->fields);
8247         free(l);
8248
8249         return -ENOMEM;
8250 }
8251
8252 static void
8253 learner_params_free(struct rte_swx_table_learner_params *params)
8254 {
8255         if (!params)
8256                 return;
8257
8258         free(params->key_mask0);
8259
8260         free(params);
8261 }
8262
8263 static struct rte_swx_table_learner_params *
8264 learner_params_get(struct learner *l)
8265 {
8266         struct rte_swx_table_learner_params *params = NULL;
8267         struct field *first, *last;
8268         uint32_t i;
8269
8270         /* Memory allocation. */
8271         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8272         if (!params)
8273                 goto error;
8274
8275         /* Find first (smallest offset) and last (biggest offset) match fields. */
8276         first = l->fields[0];
8277         last = l->fields[0];
8278
8279         for (i = 0; i < l->n_fields; i++) {
8280                 struct field *f = l->fields[i];
8281
8282                 if (f->offset < first->offset)
8283                         first = f;
8284
8285                 if (f->offset > last->offset)
8286                         last = f;
8287         }
8288
8289         /* Key offset and size. */
8290         params->key_offset = first->offset / 8;
8291         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8292
8293         /* Memory allocation. */
8294         params->key_mask0 = calloc(1, params->key_size);
8295         if (!params->key_mask0)
8296                 goto error;
8297
8298         /* Key mask. */
8299         for (i = 0; i < l->n_fields; i++) {
8300                 struct field *f = l->fields[i];
8301                 uint32_t start = (f->offset - first->offset) / 8;
8302                 size_t size = f->n_bits / 8;
8303
8304                 memset(&params->key_mask0[start], 0xFF, size);
8305         }
8306
8307         /* Action data size. */
8308         params->action_data_size = l->action_data_size_max;
8309
8310         /* Maximum number of keys. */
8311         params->n_keys_max = l->size;
8312
8313         /* Timeout. */
8314         params->key_timeout = l->timeout;
8315
8316         return params;
8317
8318 error:
8319         learner_params_free(params);
8320         return NULL;
8321 }
8322
8323 static void
8324 learner_build_free(struct rte_swx_pipeline *p)
8325 {
8326         uint32_t i;
8327
8328         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8329                 struct thread *t = &p->threads[i];
8330                 uint32_t j;
8331
8332                 if (!t->learners)
8333                         continue;
8334
8335                 for (j = 0; j < p->n_learners; j++) {
8336                         struct learner_runtime *r = &t->learners[j];
8337
8338                         free(r->mailbox);
8339                         free(r->action_data);
8340                 }
8341
8342                 free(t->learners);
8343                 t->learners = NULL;
8344         }
8345
8346         if (p->learner_stats) {
8347                 for (i = 0; i < p->n_learners; i++)
8348                         free(p->learner_stats[i].n_pkts_action);
8349
8350                 free(p->learner_stats);
8351         }
8352 }
8353
8354 static int
8355 learner_build(struct rte_swx_pipeline *p)
8356 {
8357         uint32_t i;
8358         int status = 0;
8359
8360         /* Per pipeline: learner statistics. */
8361         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8362         CHECK(p->learner_stats, ENOMEM);
8363
8364         for (i = 0; i < p->n_learners; i++) {
8365                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8366                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8367         }
8368
8369         /* Per thread: learner run-time. */
8370         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8371                 struct thread *t = &p->threads[i];
8372                 struct learner *l;
8373
8374                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8375                 if (!t->learners) {
8376                         status = -ENOMEM;
8377                         goto error;
8378                 }
8379
8380                 TAILQ_FOREACH(l, &p->learners, node) {
8381                         struct learner_runtime *r = &t->learners[l->id];
8382                         uint64_t size;
8383                         uint32_t j;
8384
8385                         /* r->mailbox. */
8386                         size = rte_swx_table_learner_mailbox_size_get();
8387                         if (size) {
8388                                 r->mailbox = calloc(1, size);
8389                                 if (!r->mailbox) {
8390                                         status = -ENOMEM;
8391                                         goto error;
8392                                 }
8393                         }
8394
8395                         /* r->key. */
8396                         r->key = l->header ?
8397                                 &t->structs[l->header->struct_id] :
8398                                 &t->structs[p->metadata_struct_id];
8399
8400                         /* r->action_data. */
8401                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8402                         if (!r->action_data) {
8403                                 status = -ENOMEM;
8404                                 goto error;
8405                         }
8406
8407                         for (j = 0; j < l->n_actions; j++) {
8408                                 struct action *a = l->actions[j];
8409                                 struct field *mf = l->action_arg[j];
8410                                 uint8_t *m = t->structs[p->metadata_struct_id];
8411
8412                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8413                         }
8414                 }
8415         }
8416
8417         return 0;
8418
8419 error:
8420         learner_build_free(p);
8421         return status;
8422 }
8423
8424 static void
8425 learner_free(struct rte_swx_pipeline *p)
8426 {
8427         learner_build_free(p);
8428
8429         /* Learner tables. */
8430         for ( ; ; ) {
8431                 struct learner *l;
8432
8433                 l = TAILQ_FIRST(&p->learners);
8434                 if (!l)
8435                         break;
8436
8437                 TAILQ_REMOVE(&p->learners, l, node);
8438                 free(l->fields);
8439                 free(l->actions);
8440                 free(l->action_arg);
8441                 free(l->default_action_data);
8442                 free(l);
8443         }
8444 }
8445
8446 /*
8447  * Table state.
8448  */
8449 static int
8450 table_state_build(struct rte_swx_pipeline *p)
8451 {
8452         struct table *table;
8453         struct selector *s;
8454         struct learner *l;
8455
8456         p->table_state = calloc(p->n_tables + p->n_selectors,
8457                                 sizeof(struct rte_swx_table_state));
8458         CHECK(p->table_state, ENOMEM);
8459
8460         TAILQ_FOREACH(table, &p->tables, node) {
8461                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8462
8463                 if (table->type) {
8464                         struct rte_swx_table_params *params;
8465
8466                         /* ts->obj. */
8467                         params = table_params_get(table);
8468                         CHECK(params, ENOMEM);
8469
8470                         ts->obj = table->type->ops.create(params,
8471                                 NULL,
8472                                 table->args,
8473                                 p->numa_node);
8474
8475                         table_params_free(params);
8476                         CHECK(ts->obj, ENODEV);
8477                 }
8478
8479                 /* ts->default_action_data. */
8480                 if (table->action_data_size_max) {
8481                         ts->default_action_data =
8482                                 malloc(table->action_data_size_max);
8483                         CHECK(ts->default_action_data, ENOMEM);
8484
8485                         memcpy(ts->default_action_data,
8486                                table->default_action_data,
8487                                table->action_data_size_max);
8488                 }
8489
8490                 /* ts->default_action_id. */
8491                 ts->default_action_id = table->default_action->id;
8492         }
8493
8494         TAILQ_FOREACH(s, &p->selectors, node) {
8495                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8496                 struct rte_swx_table_selector_params *params;
8497
8498                 /* ts->obj. */
8499                 params = selector_table_params_get(s);
8500                 CHECK(params, ENOMEM);
8501
8502                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8503
8504                 selector_params_free(params);
8505                 CHECK(ts->obj, ENODEV);
8506         }
8507
8508         TAILQ_FOREACH(l, &p->learners, node) {
8509                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8510                         p->n_selectors + l->id];
8511                 struct rte_swx_table_learner_params *params;
8512
8513                 /* ts->obj. */
8514                 params = learner_params_get(l);
8515                 CHECK(params, ENOMEM);
8516
8517                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8518                 learner_params_free(params);
8519                 CHECK(ts->obj, ENODEV);
8520
8521                 /* ts->default_action_data. */
8522                 if (l->action_data_size_max) {
8523                         ts->default_action_data = malloc(l->action_data_size_max);
8524                         CHECK(ts->default_action_data, ENOMEM);
8525
8526                         memcpy(ts->default_action_data,
8527                                l->default_action_data,
8528                                l->action_data_size_max);
8529                 }
8530
8531                 /* ts->default_action_id. */
8532                 ts->default_action_id = l->default_action->id;
8533         }
8534
8535         return 0;
8536 }
8537
8538 static void
8539 table_state_build_free(struct rte_swx_pipeline *p)
8540 {
8541         uint32_t i;
8542
8543         if (!p->table_state)
8544                 return;
8545
8546         for (i = 0; i < p->n_tables; i++) {
8547                 struct rte_swx_table_state *ts = &p->table_state[i];
8548                 struct table *table = table_find_by_id(p, i);
8549
8550                 /* ts->obj. */
8551                 if (table->type && ts->obj)
8552                         table->type->ops.free(ts->obj);
8553
8554                 /* ts->default_action_data. */
8555                 free(ts->default_action_data);
8556         }
8557
8558         for (i = 0; i < p->n_selectors; i++) {
8559                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8560
8561                 /* ts->obj. */
8562                 if (ts->obj)
8563                         rte_swx_table_selector_free(ts->obj);
8564         }
8565
8566         for (i = 0; i < p->n_learners; i++) {
8567                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8568
8569                 /* ts->obj. */
8570                 if (ts->obj)
8571                         rte_swx_table_learner_free(ts->obj);
8572
8573                 /* ts->default_action_data. */
8574                 free(ts->default_action_data);
8575         }
8576
8577         free(p->table_state);
8578         p->table_state = NULL;
8579 }
8580
8581 static void
8582 table_state_free(struct rte_swx_pipeline *p)
8583 {
8584         table_state_build_free(p);
8585 }
8586
8587 /*
8588  * Register array.
8589  */
8590 static struct regarray *
8591 regarray_find(struct rte_swx_pipeline *p, const char *name)
8592 {
8593         struct regarray *elem;
8594
8595         TAILQ_FOREACH(elem, &p->regarrays, node)
8596                 if (!strcmp(elem->name, name))
8597                         return elem;
8598
8599         return NULL;
8600 }
8601
8602 static struct regarray *
8603 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8604 {
8605         struct regarray *elem = NULL;
8606
8607         TAILQ_FOREACH(elem, &p->regarrays, node)
8608                 if (elem->id == id)
8609                         return elem;
8610
8611         return NULL;
8612 }
8613
8614 int
8615 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8616                               const char *name,
8617                               uint32_t size,
8618                               uint64_t init_val)
8619 {
8620         struct regarray *r;
8621
8622         CHECK(p, EINVAL);
8623
8624         CHECK_NAME(name, EINVAL);
8625         CHECK(!regarray_find(p, name), EEXIST);
8626
8627         CHECK(size, EINVAL);
8628         size = rte_align32pow2(size);
8629
8630         /* Memory allocation. */
8631         r = calloc(1, sizeof(struct regarray));
8632         CHECK(r, ENOMEM);
8633
8634         /* Node initialization. */
8635         strcpy(r->name, name);
8636         r->init_val = init_val;
8637         r->size = size;
8638         r->id = p->n_regarrays;
8639
8640         /* Node add to tailq. */
8641         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8642         p->n_regarrays++;
8643
8644         return 0;
8645 }
8646
8647 static int
8648 regarray_build(struct rte_swx_pipeline *p)
8649 {
8650         struct regarray *regarray;
8651
8652         if (!p->n_regarrays)
8653                 return 0;
8654
8655         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8656         CHECK(p->regarray_runtime, ENOMEM);
8657
8658         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8659                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8660                 uint32_t i;
8661
8662                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8663                                          RTE_CACHE_LINE_SIZE,
8664                                          p->numa_node);
8665                 CHECK(r->regarray, ENOMEM);
8666
8667                 if (regarray->init_val)
8668                         for (i = 0; i < regarray->size; i++)
8669                                 r->regarray[i] = regarray->init_val;
8670
8671                 r->size_mask = regarray->size - 1;
8672         }
8673
8674         return 0;
8675 }
8676
8677 static void
8678 regarray_build_free(struct rte_swx_pipeline *p)
8679 {
8680         uint32_t i;
8681
8682         if (!p->regarray_runtime)
8683                 return;
8684
8685         for (i = 0; i < p->n_regarrays; i++) {
8686                 struct regarray *regarray = regarray_find_by_id(p, i);
8687                 struct regarray_runtime *r = &p->regarray_runtime[i];
8688
8689                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8690         }
8691
8692         free(p->regarray_runtime);
8693         p->regarray_runtime = NULL;
8694 }
8695
8696 static void
8697 regarray_free(struct rte_swx_pipeline *p)
8698 {
8699         regarray_build_free(p);
8700
8701         for ( ; ; ) {
8702                 struct regarray *elem;
8703
8704                 elem = TAILQ_FIRST(&p->regarrays);
8705                 if (!elem)
8706                         break;
8707
8708                 TAILQ_REMOVE(&p->regarrays, elem, node);
8709                 free(elem);
8710         }
8711 }
8712
8713 /*
8714  * Meter array.
8715  */
8716 static struct meter_profile *
8717 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8718 {
8719         struct meter_profile *elem;
8720
8721         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8722                 if (!strcmp(elem->name, name))
8723                         return elem;
8724
8725         return NULL;
8726 }
8727
8728 static struct metarray *
8729 metarray_find(struct rte_swx_pipeline *p, const char *name)
8730 {
8731         struct metarray *elem;
8732
8733         TAILQ_FOREACH(elem, &p->metarrays, node)
8734                 if (!strcmp(elem->name, name))
8735                         return elem;
8736
8737         return NULL;
8738 }
8739
8740 static struct metarray *
8741 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8742 {
8743         struct metarray *elem = NULL;
8744
8745         TAILQ_FOREACH(elem, &p->metarrays, node)
8746                 if (elem->id == id)
8747                         return elem;
8748
8749         return NULL;
8750 }
8751
8752 int
8753 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8754                                  const char *name,
8755                                  uint32_t size)
8756 {
8757         struct metarray *m;
8758
8759         CHECK(p, EINVAL);
8760
8761         CHECK_NAME(name, EINVAL);
8762         CHECK(!metarray_find(p, name), EEXIST);
8763
8764         CHECK(size, EINVAL);
8765         size = rte_align32pow2(size);
8766
8767         /* Memory allocation. */
8768         m = calloc(1, sizeof(struct metarray));
8769         CHECK(m, ENOMEM);
8770
8771         /* Node initialization. */
8772         strcpy(m->name, name);
8773         m->size = size;
8774         m->id = p->n_metarrays;
8775
8776         /* Node add to tailq. */
8777         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8778         p->n_metarrays++;
8779
8780         return 0;
8781 }
8782
8783 struct meter_profile meter_profile_default = {
8784         .node = {0},
8785         .name = "",
8786         .params = {0},
8787
8788         .profile = {
8789                 .cbs = 10000,
8790                 .pbs = 10000,
8791                 .cir_period = 1,
8792                 .cir_bytes_per_period = 1,
8793                 .pir_period = 1,
8794                 .pir_bytes_per_period = 1,
8795         },
8796
8797         .n_users = 0,
8798 };
8799
8800 static void
8801 meter_init(struct meter *m)
8802 {
8803         memset(m, 0, sizeof(struct meter));
8804         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8805         m->profile = &meter_profile_default;
8806         m->color_mask = RTE_COLOR_GREEN;
8807
8808         meter_profile_default.n_users++;
8809 }
8810
8811 static int
8812 metarray_build(struct rte_swx_pipeline *p)
8813 {
8814         struct metarray *m;
8815
8816         if (!p->n_metarrays)
8817                 return 0;
8818
8819         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8820         CHECK(p->metarray_runtime, ENOMEM);
8821
8822         TAILQ_FOREACH(m, &p->metarrays, node) {
8823                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8824                 uint32_t i;
8825
8826                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8827                                          RTE_CACHE_LINE_SIZE,
8828                                          p->numa_node);
8829                 CHECK(r->metarray, ENOMEM);
8830
8831                 for (i = 0; i < m->size; i++)
8832                         meter_init(&r->metarray[i]);
8833
8834                 r->size_mask = m->size - 1;
8835         }
8836
8837         return 0;
8838 }
8839
8840 static void
8841 metarray_build_free(struct rte_swx_pipeline *p)
8842 {
8843         uint32_t i;
8844
8845         if (!p->metarray_runtime)
8846                 return;
8847
8848         for (i = 0; i < p->n_metarrays; i++) {
8849                 struct metarray *m = metarray_find_by_id(p, i);
8850                 struct metarray_runtime *r = &p->metarray_runtime[i];
8851
8852                 env_free(r->metarray, m->size * sizeof(struct meter));
8853         }
8854
8855         free(p->metarray_runtime);
8856         p->metarray_runtime = NULL;
8857 }
8858
8859 static void
8860 metarray_free(struct rte_swx_pipeline *p)
8861 {
8862         metarray_build_free(p);
8863
8864         /* Meter arrays. */
8865         for ( ; ; ) {
8866                 struct metarray *elem;
8867
8868                 elem = TAILQ_FIRST(&p->metarrays);
8869                 if (!elem)
8870                         break;
8871
8872                 TAILQ_REMOVE(&p->metarrays, elem, node);
8873                 free(elem);
8874         }
8875
8876         /* Meter profiles. */
8877         for ( ; ; ) {
8878                 struct meter_profile *elem;
8879
8880                 elem = TAILQ_FIRST(&p->meter_profiles);
8881                 if (!elem)
8882                         break;
8883
8884                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8885                 free(elem);
8886         }
8887 }
8888
8889 /*
8890  * Pipeline.
8891  */
8892 int
8893 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8894 {
8895         struct rte_swx_pipeline *pipeline;
8896
8897         /* Check input parameters. */
8898         CHECK(p, EINVAL);
8899
8900         /* Memory allocation. */
8901         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8902         CHECK(pipeline, ENOMEM);
8903
8904         /* Initialization. */
8905         TAILQ_INIT(&pipeline->struct_types);
8906         TAILQ_INIT(&pipeline->port_in_types);
8907         TAILQ_INIT(&pipeline->ports_in);
8908         TAILQ_INIT(&pipeline->port_out_types);
8909         TAILQ_INIT(&pipeline->ports_out);
8910         TAILQ_INIT(&pipeline->extern_types);
8911         TAILQ_INIT(&pipeline->extern_objs);
8912         TAILQ_INIT(&pipeline->extern_funcs);
8913         TAILQ_INIT(&pipeline->headers);
8914         TAILQ_INIT(&pipeline->actions);
8915         TAILQ_INIT(&pipeline->table_types);
8916         TAILQ_INIT(&pipeline->tables);
8917         TAILQ_INIT(&pipeline->selectors);
8918         TAILQ_INIT(&pipeline->learners);
8919         TAILQ_INIT(&pipeline->regarrays);
8920         TAILQ_INIT(&pipeline->meter_profiles);
8921         TAILQ_INIT(&pipeline->metarrays);
8922
8923         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8924         pipeline->numa_node = numa_node;
8925
8926         *p = pipeline;
8927         return 0;
8928 }
8929
8930 void
8931 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8932 {
8933         if (!p)
8934                 return;
8935
8936         free(p->instruction_data);
8937         free(p->instructions);
8938
8939         metarray_free(p);
8940         regarray_free(p);
8941         table_state_free(p);
8942         learner_free(p);
8943         selector_free(p);
8944         table_free(p);
8945         action_free(p);
8946         instruction_table_free(p);
8947         metadata_free(p);
8948         header_free(p);
8949         extern_func_free(p);
8950         extern_obj_free(p);
8951         port_out_free(p);
8952         port_in_free(p);
8953         struct_free(p);
8954
8955         free(p);
8956 }
8957
8958 int
8959 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
8960                                      const char **instructions,
8961                                      uint32_t n_instructions)
8962 {
8963         int err;
8964         uint32_t i;
8965
8966         err = instruction_config(p, NULL, instructions, n_instructions);
8967         if (err)
8968                 return err;
8969
8970         /* Thread instruction pointer reset. */
8971         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8972                 struct thread *t = &p->threads[i];
8973
8974                 thread_ip_reset(p, t);
8975         }
8976
8977         return 0;
8978 }
8979
8980 int
8981 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
8982 {
8983         int status;
8984
8985         CHECK(p, EINVAL);
8986         CHECK(p->build_done == 0, EEXIST);
8987
8988         status = port_in_build(p);
8989         if (status)
8990                 goto error;
8991
8992         status = port_out_build(p);
8993         if (status)
8994                 goto error;
8995
8996         status = struct_build(p);
8997         if (status)
8998                 goto error;
8999
9000         status = extern_obj_build(p);
9001         if (status)
9002                 goto error;
9003
9004         status = extern_func_build(p);
9005         if (status)
9006                 goto error;
9007
9008         status = header_build(p);
9009         if (status)
9010                 goto error;
9011
9012         status = metadata_build(p);
9013         if (status)
9014                 goto error;
9015
9016         status = instruction_table_build(p);
9017         if (status)
9018                 goto error;
9019
9020         status = action_build(p);
9021         if (status)
9022                 goto error;
9023
9024         status = table_build(p);
9025         if (status)
9026                 goto error;
9027
9028         status = selector_build(p);
9029         if (status)
9030                 goto error;
9031
9032         status = learner_build(p);
9033         if (status)
9034                 goto error;
9035
9036         status = table_state_build(p);
9037         if (status)
9038                 goto error;
9039
9040         status = regarray_build(p);
9041         if (status)
9042                 goto error;
9043
9044         status = metarray_build(p);
9045         if (status)
9046                 goto error;
9047
9048         p->build_done = 1;
9049         return 0;
9050
9051 error:
9052         metarray_build_free(p);
9053         regarray_build_free(p);
9054         table_state_build_free(p);
9055         learner_build_free(p);
9056         selector_build_free(p);
9057         table_build_free(p);
9058         action_build_free(p);
9059         instruction_table_build_free(p);
9060         metadata_build_free(p);
9061         header_build_free(p);
9062         extern_func_build_free(p);
9063         extern_obj_build_free(p);
9064         port_out_build_free(p);
9065         port_in_build_free(p);
9066         struct_build_free(p);
9067
9068         return status;
9069 }
9070
9071 void
9072 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9073 {
9074         uint32_t i;
9075
9076         for (i = 0; i < n_instructions; i++)
9077                 instr_exec(p);
9078 }
9079
9080 void
9081 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9082 {
9083         uint32_t i;
9084
9085         for (i = 0; i < p->n_ports_out; i++) {
9086                 struct port_out_runtime *port = &p->out[i];
9087
9088                 if (port->flush)
9089                         port->flush(port->obj);
9090         }
9091 }
9092
9093 /*
9094  * Control.
9095  */
9096 int
9097 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9098                               struct rte_swx_ctl_pipeline_info *pipeline)
9099 {
9100         struct action *action;
9101         struct table *table;
9102         uint32_t n_actions = 0, n_tables = 0;
9103
9104         if (!p || !pipeline)
9105                 return -EINVAL;
9106
9107         TAILQ_FOREACH(action, &p->actions, node)
9108                 n_actions++;
9109
9110         TAILQ_FOREACH(table, &p->tables, node)
9111                 n_tables++;
9112
9113         pipeline->n_ports_in = p->n_ports_in;
9114         pipeline->n_ports_out = p->n_ports_out;
9115         pipeline->n_actions = n_actions;
9116         pipeline->n_tables = n_tables;
9117         pipeline->n_selectors = p->n_selectors;
9118         pipeline->n_learners = p->n_learners;
9119         pipeline->n_regarrays = p->n_regarrays;
9120         pipeline->n_metarrays = p->n_metarrays;
9121
9122         return 0;
9123 }
9124
9125 int
9126 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9127 {
9128         if (!p || !numa_node)
9129                 return -EINVAL;
9130
9131         *numa_node = p->numa_node;
9132         return 0;
9133 }
9134
9135 int
9136 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9137                             uint32_t action_id,
9138                             struct rte_swx_ctl_action_info *action)
9139 {
9140         struct action *a = NULL;
9141
9142         if (!p || (action_id >= p->n_actions) || !action)
9143                 return -EINVAL;
9144
9145         a = action_find_by_id(p, action_id);
9146         if (!a)
9147                 return -EINVAL;
9148
9149         strcpy(action->name, a->name);
9150         action->n_args = a->st ? a->st->n_fields : 0;
9151         return 0;
9152 }
9153
9154 int
9155 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9156                                 uint32_t action_id,
9157                                 uint32_t action_arg_id,
9158                                 struct rte_swx_ctl_action_arg_info *action_arg)
9159 {
9160         struct action *a = NULL;
9161         struct field *arg = NULL;
9162
9163         if (!p || (action_id >= p->n_actions) || !action_arg)
9164                 return -EINVAL;
9165
9166         a = action_find_by_id(p, action_id);
9167         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9168                 return -EINVAL;
9169
9170         arg = &a->st->fields[action_arg_id];
9171         strcpy(action_arg->name, arg->name);
9172         action_arg->n_bits = arg->n_bits;
9173         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9174
9175         return 0;
9176 }
9177
9178 int
9179 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9180                            uint32_t table_id,
9181                            struct rte_swx_ctl_table_info *table)
9182 {
9183         struct table *t = NULL;
9184
9185         if (!p || !table)
9186                 return -EINVAL;
9187
9188         t = table_find_by_id(p, table_id);
9189         if (!t)
9190                 return -EINVAL;
9191
9192         strcpy(table->name, t->name);
9193         strcpy(table->args, t->args);
9194         table->n_match_fields = t->n_fields;
9195         table->n_actions = t->n_actions;
9196         table->default_action_is_const = t->default_action_is_const;
9197         table->size = t->size;
9198         return 0;
9199 }
9200
9201 int
9202 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9203         uint32_t table_id,
9204         uint32_t match_field_id,
9205         struct rte_swx_ctl_table_match_field_info *match_field)
9206 {
9207         struct table *t;
9208         struct match_field *f;
9209
9210         if (!p || (table_id >= p->n_tables) || !match_field)
9211                 return -EINVAL;
9212
9213         t = table_find_by_id(p, table_id);
9214         if (!t || (match_field_id >= t->n_fields))
9215                 return -EINVAL;
9216
9217         f = &t->fields[match_field_id];
9218         match_field->match_type = f->match_type;
9219         match_field->is_header = t->header ? 1 : 0;
9220         match_field->n_bits = f->field->n_bits;
9221         match_field->offset = f->field->offset;
9222
9223         return 0;
9224 }
9225
9226 int
9227 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9228         uint32_t table_id,
9229         uint32_t table_action_id,
9230         struct rte_swx_ctl_table_action_info *table_action)
9231 {
9232         struct table *t;
9233
9234         if (!p || (table_id >= p->n_tables) || !table_action)
9235                 return -EINVAL;
9236
9237         t = table_find_by_id(p, table_id);
9238         if (!t || (table_action_id >= t->n_actions))
9239                 return -EINVAL;
9240
9241         table_action->action_id = t->actions[table_action_id]->id;
9242
9243         return 0;
9244 }
9245
9246 int
9247 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9248                           uint32_t table_id,
9249                           struct rte_swx_table_ops *table_ops,
9250                           int *is_stub)
9251 {
9252         struct table *t;
9253
9254         if (!p || (table_id >= p->n_tables))
9255                 return -EINVAL;
9256
9257         t = table_find_by_id(p, table_id);
9258         if (!t)
9259                 return -EINVAL;
9260
9261         if (t->type) {
9262                 if (table_ops)
9263                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9264                 *is_stub = 0;
9265         } else {
9266                 *is_stub = 1;
9267         }
9268
9269         return 0;
9270 }
9271
9272 int
9273 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9274                               uint32_t selector_id,
9275                               struct rte_swx_ctl_selector_info *selector)
9276 {
9277         struct selector *s = NULL;
9278
9279         if (!p || !selector)
9280                 return -EINVAL;
9281
9282         s = selector_find_by_id(p, selector_id);
9283         if (!s)
9284                 return -EINVAL;
9285
9286         strcpy(selector->name, s->name);
9287
9288         selector->n_selector_fields = s->n_selector_fields;
9289         selector->n_groups_max = s->n_groups_max;
9290         selector->n_members_per_group_max = s->n_members_per_group_max;
9291
9292         return 0;
9293 }
9294
9295 int
9296 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9297          uint32_t selector_id,
9298          struct rte_swx_ctl_table_match_field_info *field)
9299 {
9300         struct selector *s;
9301
9302         if (!p || (selector_id >= p->n_selectors) || !field)
9303                 return -EINVAL;
9304
9305         s = selector_find_by_id(p, selector_id);
9306         if (!s)
9307                 return -EINVAL;
9308
9309         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9310         field->is_header = 0;
9311         field->n_bits = s->group_id_field->n_bits;
9312         field->offset = s->group_id_field->offset;
9313
9314         return 0;
9315 }
9316
9317 int
9318 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9319          uint32_t selector_id,
9320          uint32_t selector_field_id,
9321          struct rte_swx_ctl_table_match_field_info *field)
9322 {
9323         struct selector *s;
9324         struct field *f;
9325
9326         if (!p || (selector_id >= p->n_selectors) || !field)
9327                 return -EINVAL;
9328
9329         s = selector_find_by_id(p, selector_id);
9330         if (!s || (selector_field_id >= s->n_selector_fields))
9331                 return -EINVAL;
9332
9333         f = s->selector_fields[selector_field_id];
9334         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9335         field->is_header = s->selector_header ? 1 : 0;
9336         field->n_bits = f->n_bits;
9337         field->offset = f->offset;
9338
9339         return 0;
9340 }
9341
9342 int
9343 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9344          uint32_t selector_id,
9345          struct rte_swx_ctl_table_match_field_info *field)
9346 {
9347         struct selector *s;
9348
9349         if (!p || (selector_id >= p->n_selectors) || !field)
9350                 return -EINVAL;
9351
9352         s = selector_find_by_id(p, selector_id);
9353         if (!s)
9354                 return -EINVAL;
9355
9356         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9357         field->is_header = 0;
9358         field->n_bits = s->member_id_field->n_bits;
9359         field->offset = s->member_id_field->offset;
9360
9361         return 0;
9362 }
9363
9364 int
9365 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9366                              uint32_t learner_id,
9367                              struct rte_swx_ctl_learner_info *learner)
9368 {
9369         struct learner *l = NULL;
9370
9371         if (!p || !learner)
9372                 return -EINVAL;
9373
9374         l = learner_find_by_id(p, learner_id);
9375         if (!l)
9376                 return -EINVAL;
9377
9378         strcpy(learner->name, l->name);
9379
9380         learner->n_match_fields = l->n_fields;
9381         learner->n_actions = l->n_actions;
9382         learner->default_action_is_const = l->default_action_is_const;
9383         learner->size = l->size;
9384
9385         return 0;
9386 }
9387
9388 int
9389 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9390                                          uint32_t learner_id,
9391                                          uint32_t match_field_id,
9392                                          struct rte_swx_ctl_table_match_field_info *match_field)
9393 {
9394         struct learner *l;
9395         struct field *f;
9396
9397         if (!p || (learner_id >= p->n_learners) || !match_field)
9398                 return -EINVAL;
9399
9400         l = learner_find_by_id(p, learner_id);
9401         if (!l || (match_field_id >= l->n_fields))
9402                 return -EINVAL;
9403
9404         f = l->fields[match_field_id];
9405         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9406         match_field->is_header = l->header ? 1 : 0;
9407         match_field->n_bits = f->n_bits;
9408         match_field->offset = f->offset;
9409
9410         return 0;
9411 }
9412
9413 int
9414 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9415                                     uint32_t learner_id,
9416                                     uint32_t learner_action_id,
9417                                     struct rte_swx_ctl_table_action_info *learner_action)
9418 {
9419         struct learner *l;
9420
9421         if (!p || (learner_id >= p->n_learners) || !learner_action)
9422                 return -EINVAL;
9423
9424         l = learner_find_by_id(p, learner_id);
9425         if (!l || (learner_action_id >= l->n_actions))
9426                 return -EINVAL;
9427
9428         learner_action->action_id = l->actions[learner_action_id]->id;
9429
9430         return 0;
9431 }
9432
9433 int
9434 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9435                                  struct rte_swx_table_state **table_state)
9436 {
9437         if (!p || !table_state || !p->build_done)
9438                 return -EINVAL;
9439
9440         *table_state = p->table_state;
9441         return 0;
9442 }
9443
9444 int
9445 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9446                                  struct rte_swx_table_state *table_state)
9447 {
9448         if (!p || !table_state || !p->build_done)
9449                 return -EINVAL;
9450
9451         p->table_state = table_state;
9452         return 0;
9453 }
9454
9455 int
9456 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9457                                         uint32_t port_id,
9458                                         struct rte_swx_port_in_stats *stats)
9459 {
9460         struct port_in *port;
9461
9462         if (!p || !stats)
9463                 return -EINVAL;
9464
9465         port = port_in_find(p, port_id);
9466         if (!port)
9467                 return -EINVAL;
9468
9469         port->type->ops.stats_read(port->obj, stats);
9470         return 0;
9471 }
9472
9473 int
9474 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9475                                          uint32_t port_id,
9476                                          struct rte_swx_port_out_stats *stats)
9477 {
9478         struct port_out *port;
9479
9480         if (!p || !stats)
9481                 return -EINVAL;
9482
9483         port = port_out_find(p, port_id);
9484         if (!port)
9485                 return -EINVAL;
9486
9487         port->type->ops.stats_read(port->obj, stats);
9488         return 0;
9489 }
9490
9491 int
9492 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9493                                       const char *table_name,
9494                                       struct rte_swx_table_stats *stats)
9495 {
9496         struct table *table;
9497         struct table_statistics *table_stats;
9498
9499         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9500                 return -EINVAL;
9501
9502         table = table_find(p, table_name);
9503         if (!table)
9504                 return -EINVAL;
9505
9506         table_stats = &p->table_stats[table->id];
9507
9508         memcpy(stats->n_pkts_action,
9509                table_stats->n_pkts_action,
9510                p->n_actions * sizeof(uint64_t));
9511
9512         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9513         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9514
9515         return 0;
9516 }
9517
9518 int
9519 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9520         const char *selector_name,
9521         struct rte_swx_pipeline_selector_stats *stats)
9522 {
9523         struct selector *s;
9524
9525         if (!p || !selector_name || !selector_name[0] || !stats)
9526                 return -EINVAL;
9527
9528         s = selector_find(p, selector_name);
9529         if (!s)
9530                 return -EINVAL;
9531
9532         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9533
9534         return 0;
9535 }
9536
9537 int
9538 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9539                                         const char *learner_name,
9540                                         struct rte_swx_learner_stats *stats)
9541 {
9542         struct learner *l;
9543         struct learner_statistics *learner_stats;
9544
9545         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9546                 return -EINVAL;
9547
9548         l = learner_find(p, learner_name);
9549         if (!l)
9550                 return -EINVAL;
9551
9552         learner_stats = &p->learner_stats[l->id];
9553
9554         memcpy(stats->n_pkts_action,
9555                learner_stats->n_pkts_action,
9556                p->n_actions * sizeof(uint64_t));
9557
9558         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9559         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9560
9561         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9562         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9563
9564         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9565
9566         return 0;
9567 }
9568
9569 int
9570 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9571                               uint32_t regarray_id,
9572                               struct rte_swx_ctl_regarray_info *regarray)
9573 {
9574         struct regarray *r;
9575
9576         if (!p || !regarray)
9577                 return -EINVAL;
9578
9579         r = regarray_find_by_id(p, regarray_id);
9580         if (!r)
9581                 return -EINVAL;
9582
9583         strcpy(regarray->name, r->name);
9584         regarray->size = r->size;
9585         return 0;
9586 }
9587
9588 int
9589 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9590                                    const char *regarray_name,
9591                                    uint32_t regarray_index,
9592                                    uint64_t *value)
9593 {
9594         struct regarray *regarray;
9595         struct regarray_runtime *r;
9596
9597         if (!p || !regarray_name || !value)
9598                 return -EINVAL;
9599
9600         regarray = regarray_find(p, regarray_name);
9601         if (!regarray || (regarray_index >= regarray->size))
9602                 return -EINVAL;
9603
9604         r = &p->regarray_runtime[regarray->id];
9605         *value = r->regarray[regarray_index];
9606         return 0;
9607 }
9608
9609 int
9610 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9611                                    const char *regarray_name,
9612                                    uint32_t regarray_index,
9613                                    uint64_t value)
9614 {
9615         struct regarray *regarray;
9616         struct regarray_runtime *r;
9617
9618         if (!p || !regarray_name)
9619                 return -EINVAL;
9620
9621         regarray = regarray_find(p, regarray_name);
9622         if (!regarray || (regarray_index >= regarray->size))
9623                 return -EINVAL;
9624
9625         r = &p->regarray_runtime[regarray->id];
9626         r->regarray[regarray_index] = value;
9627         return 0;
9628 }
9629
9630 int
9631 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9632                               uint32_t metarray_id,
9633                               struct rte_swx_ctl_metarray_info *metarray)
9634 {
9635         struct metarray *m;
9636
9637         if (!p || !metarray)
9638                 return -EINVAL;
9639
9640         m = metarray_find_by_id(p, metarray_id);
9641         if (!m)
9642                 return -EINVAL;
9643
9644         strcpy(metarray->name, m->name);
9645         metarray->size = m->size;
9646         return 0;
9647 }
9648
9649 int
9650 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9651                               const char *name,
9652                               struct rte_meter_trtcm_params *params)
9653 {
9654         struct meter_profile *mp;
9655         int status;
9656
9657         CHECK(p, EINVAL);
9658         CHECK_NAME(name, EINVAL);
9659         CHECK(params, EINVAL);
9660         CHECK(!meter_profile_find(p, name), EEXIST);
9661
9662         /* Node allocation. */
9663         mp = calloc(1, sizeof(struct meter_profile));
9664         CHECK(mp, ENOMEM);
9665
9666         /* Node initialization. */
9667         strcpy(mp->name, name);
9668         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9669         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9670         if (status) {
9671                 free(mp);
9672                 CHECK(0, EINVAL);
9673         }
9674
9675         /* Node add to tailq. */
9676         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9677
9678         return 0;
9679 }
9680
9681 int
9682 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9683                                  const char *name)
9684 {
9685         struct meter_profile *mp;
9686
9687         CHECK(p, EINVAL);
9688         CHECK_NAME(name, EINVAL);
9689
9690         mp = meter_profile_find(p, name);
9691         CHECK(mp, EINVAL);
9692         CHECK(!mp->n_users, EBUSY);
9693
9694         /* Remove node from tailq. */
9695         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9696         free(mp);
9697
9698         return 0;
9699 }
9700
9701 int
9702 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9703                         const char *metarray_name,
9704                         uint32_t metarray_index)
9705 {
9706         struct meter_profile *mp_old;
9707         struct metarray *metarray;
9708         struct metarray_runtime *metarray_runtime;
9709         struct meter *m;
9710
9711         CHECK(p, EINVAL);
9712         CHECK_NAME(metarray_name, EINVAL);
9713
9714         metarray = metarray_find(p, metarray_name);
9715         CHECK(metarray, EINVAL);
9716         CHECK(metarray_index < metarray->size, EINVAL);
9717
9718         metarray_runtime = &p->metarray_runtime[metarray->id];
9719         m = &metarray_runtime->metarray[metarray_index];
9720         mp_old = m->profile;
9721
9722         meter_init(m);
9723
9724         mp_old->n_users--;
9725
9726         return 0;
9727 }
9728
9729 int
9730 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9731                       const char *metarray_name,
9732                       uint32_t metarray_index,
9733                       const char *profile_name)
9734 {
9735         struct meter_profile *mp, *mp_old;
9736         struct metarray *metarray;
9737         struct metarray_runtime *metarray_runtime;
9738         struct meter *m;
9739
9740         CHECK(p, EINVAL);
9741         CHECK_NAME(metarray_name, EINVAL);
9742
9743         metarray = metarray_find(p, metarray_name);
9744         CHECK(metarray, EINVAL);
9745         CHECK(metarray_index < metarray->size, EINVAL);
9746
9747         mp = meter_profile_find(p, profile_name);
9748         CHECK(mp, EINVAL);
9749
9750         metarray_runtime = &p->metarray_runtime[metarray->id];
9751         m = &metarray_runtime->metarray[metarray_index];
9752         mp_old = m->profile;
9753
9754         memset(m, 0, sizeof(struct meter));
9755         rte_meter_trtcm_config(&m->m, &mp->profile);
9756         m->profile = mp;
9757         m->color_mask = RTE_COLORS;
9758
9759         mp->n_users++;
9760         mp_old->n_users--;
9761
9762         return 0;
9763 }
9764
9765 int
9766 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9767                              const char *metarray_name,
9768                              uint32_t metarray_index,
9769                              struct rte_swx_ctl_meter_stats *stats)
9770 {
9771         struct metarray *metarray;
9772         struct metarray_runtime *metarray_runtime;
9773         struct meter *m;
9774
9775         CHECK(p, EINVAL);
9776         CHECK_NAME(metarray_name, EINVAL);
9777
9778         metarray = metarray_find(p, metarray_name);
9779         CHECK(metarray, EINVAL);
9780         CHECK(metarray_index < metarray->size, EINVAL);
9781
9782         CHECK(stats, EINVAL);
9783
9784         metarray_runtime = &p->metarray_runtime[metarray->id];
9785         m = &metarray_runtime->metarray[metarray_index];
9786
9787         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9788         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9789
9790         return 0;
9791 }