lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t selector_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107         struct selector_runtime *selector = &t->selectors[selector_id];
2108         struct selector_statistics *stats = &p->selector_stats[selector_id];
2109         uint64_t n_pkts = stats->n_pkts;
2110         int done;
2111
2112         /* Table. */
2113         done = rte_swx_table_selector_select(ts->obj,
2114                            selector->mailbox,
2115                            selector->group_id_buffer,
2116                            selector->selector_buffer,
2117                            selector->member_id_buffer);
2118         if (!done) {
2119                 /* Thread. */
2120                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2121                       p->thread_id,
2122                       selector_id);
2123
2124                 thread_yield(p);
2125                 return;
2126         }
2127
2128
2129         TRACE("[Thread %2u] selector %u\n",
2130               p->thread_id,
2131               selector_id);
2132
2133         stats->n_pkts = n_pkts + 1;
2134
2135         /* Thread. */
2136         thread_ip_inc(p);
2137 }
2138
2139 static inline void
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t learner_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146                 p->n_selectors + learner_id];
2147         struct learner_runtime *l = &t->learners[learner_id];
2148         struct learner_statistics *stats = &p->learner_stats[learner_id];
2149         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150         uint8_t *action_data;
2151         int done, hit;
2152
2153         /* Table. */
2154         time = rte_get_tsc_cycles();
2155
2156         done = rte_swx_table_learner_lookup(ts->obj,
2157                                             l->mailbox,
2158                                             time,
2159                                             l->key,
2160                                             &action_id,
2161                                             &action_data,
2162                                             &hit);
2163         if (!done) {
2164                 /* Thread. */
2165                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2166                       p->thread_id,
2167                       learner_id);
2168
2169                 thread_yield(p);
2170                 return;
2171         }
2172
2173         action_id = hit ? action_id : ts->default_action_id;
2174         action_data = hit ? action_data : ts->default_action_data;
2175         n_pkts_hit = stats->n_pkts_hit[hit];
2176         n_pkts_action = stats->n_pkts_action[action_id];
2177
2178         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2179               p->thread_id,
2180               learner_id,
2181               hit ? "hit" : "miss",
2182               (uint32_t)action_id);
2183
2184         t->action_id = action_id;
2185         t->structs[0] = action_data;
2186         t->hit = hit;
2187         t->learner_id = learner_id;
2188         t->time = time;
2189         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2191
2192         /* Thread. */
2193         thread_ip_action_call(p, t, action_id);
2194 }
2195
2196 /*
2197  * learn.
2198  */
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2201
2202 static int
2203 action_has_nbo_args(struct action *a);
2204
2205 static int
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207                       struct action *action,
2208                       char **tokens,
2209                       int n_tokens,
2210                       struct instruction *instr,
2211                       struct instruction_data *data __rte_unused)
2212 {
2213         struct action *a;
2214
2215         CHECK(action, EINVAL);
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         a = action_find(p, tokens[1]);
2219         CHECK(a, EINVAL);
2220         CHECK(!action_has_nbo_args(a), EINVAL);
2221
2222         instr->type = INSTR_LEARNER_LEARN;
2223         instr->learn.action_id = a->id;
2224
2225         return 0;
2226 }
2227
2228 static inline void
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_learn_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_inc(p);
2238 }
2239
2240 /*
2241  * forget.
2242  */
2243 static int
2244 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2245                        struct action *action,
2246                        char **tokens __rte_unused,
2247                        int n_tokens,
2248                        struct instruction *instr,
2249                        struct instruction_data *data __rte_unused)
2250 {
2251         CHECK(action, EINVAL);
2252         CHECK(n_tokens == 1, EINVAL);
2253
2254         instr->type = INSTR_LEARNER_FORGET;
2255
2256         return 0;
2257 }
2258
2259 static inline void
2260 instr_forget_exec(struct rte_swx_pipeline *p)
2261 {
2262         struct thread *t = &p->threads[p->thread_id];
2263         struct instruction *ip = t->ip;
2264
2265         __instr_forget_exec(p, t, ip);
2266
2267         /* Thread. */
2268         thread_ip_inc(p);
2269 }
2270
2271 /*
2272  * extern.
2273  */
2274 static int
2275 instr_extern_translate(struct rte_swx_pipeline *p,
2276                        struct action *action __rte_unused,
2277                        char **tokens,
2278                        int n_tokens,
2279                        struct instruction *instr,
2280                        struct instruction_data *data __rte_unused)
2281 {
2282         char *token = tokens[1];
2283
2284         CHECK(n_tokens == 2, EINVAL);
2285
2286         if (token[0] == 'e') {
2287                 struct extern_obj *obj;
2288                 struct extern_type_member_func *func;
2289
2290                 func = extern_obj_member_func_parse(p, token, &obj);
2291                 CHECK(func, EINVAL);
2292
2293                 instr->type = INSTR_EXTERN_OBJ;
2294                 instr->ext_obj.ext_obj_id = obj->id;
2295                 instr->ext_obj.func_id = func->id;
2296
2297                 return 0;
2298         }
2299
2300         if (token[0] == 'f') {
2301                 struct extern_func *func;
2302
2303                 func = extern_func_parse(p, token);
2304                 CHECK(func, EINVAL);
2305
2306                 instr->type = INSTR_EXTERN_FUNC;
2307                 instr->ext_func.ext_func_id = func->id;
2308
2309                 return 0;
2310         }
2311
2312         CHECK(0, EINVAL);
2313 }
2314
2315 static inline void
2316 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2317 {
2318         struct thread *t = &p->threads[p->thread_id];
2319         struct instruction *ip = t->ip;
2320         uint32_t done;
2321
2322         /* Extern object member function execute. */
2323         done = __instr_extern_obj_exec(p, t, ip);
2324
2325         /* Thread. */
2326         thread_ip_inc_cond(t, done);
2327         thread_yield_cond(p, done ^ 1);
2328 }
2329
2330 static inline void
2331 instr_extern_func_exec(struct rte_swx_pipeline *p)
2332 {
2333         struct thread *t = &p->threads[p->thread_id];
2334         struct instruction *ip = t->ip;
2335         uint32_t done;
2336
2337         /* Extern function execute. */
2338         done = __instr_extern_func_exec(p, t, ip);
2339
2340         /* Thread. */
2341         thread_ip_inc_cond(t, done);
2342         thread_yield_cond(p, done ^ 1);
2343 }
2344
2345 /*
2346  * mov.
2347  */
2348 static int
2349 instr_mov_translate(struct rte_swx_pipeline *p,
2350                     struct action *action,
2351                     char **tokens,
2352                     int n_tokens,
2353                     struct instruction *instr,
2354                     struct instruction_data *data __rte_unused)
2355 {
2356         char *dst = tokens[1], *src = tokens[2];
2357         struct field *fdst, *fsrc;
2358         uint64_t src_val;
2359         uint32_t dst_struct_id = 0, src_struct_id = 0;
2360
2361         CHECK(n_tokens == 3, EINVAL);
2362
2363         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2364         CHECK(fdst, EINVAL);
2365         CHECK(!fdst->var_size, EINVAL);
2366
2367         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2368         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2369         if (fsrc) {
2370                 CHECK(!fsrc->var_size, EINVAL);
2371
2372                 instr->type = INSTR_MOV;
2373                 if (dst[0] != 'h' && src[0] == 'h')
2374                         instr->type = INSTR_MOV_MH;
2375                 if (dst[0] == 'h' && src[0] != 'h')
2376                         instr->type = INSTR_MOV_HM;
2377                 if (dst[0] == 'h' && src[0] == 'h')
2378                         instr->type = INSTR_MOV_HH;
2379
2380                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2381                 instr->mov.dst.n_bits = fdst->n_bits;
2382                 instr->mov.dst.offset = fdst->offset / 8;
2383                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2384                 instr->mov.src.n_bits = fsrc->n_bits;
2385                 instr->mov.src.offset = fsrc->offset / 8;
2386                 return 0;
2387         }
2388
2389         /* MOV_I. */
2390         src_val = strtoull(src, &src, 0);
2391         CHECK(!src[0], EINVAL);
2392
2393         if (dst[0] == 'h')
2394                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2395
2396         instr->type = INSTR_MOV_I;
2397         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2398         instr->mov.dst.n_bits = fdst->n_bits;
2399         instr->mov.dst.offset = fdst->offset / 8;
2400         instr->mov.src_val = src_val;
2401         return 0;
2402 }
2403
2404 static inline void
2405 instr_mov_exec(struct rte_swx_pipeline *p)
2406 {
2407         struct thread *t = &p->threads[p->thread_id];
2408         struct instruction *ip = t->ip;
2409
2410         __instr_mov_exec(p, t, ip);
2411
2412         /* Thread. */
2413         thread_ip_inc(p);
2414 }
2415
2416 static inline void
2417 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2418 {
2419         struct thread *t = &p->threads[p->thread_id];
2420         struct instruction *ip = t->ip;
2421
2422         __instr_mov_mh_exec(p, t, ip);
2423
2424         /* Thread. */
2425         thread_ip_inc(p);
2426 }
2427
2428 static inline void
2429 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2430 {
2431         struct thread *t = &p->threads[p->thread_id];
2432         struct instruction *ip = t->ip;
2433
2434         __instr_mov_hm_exec(p, t, ip);
2435
2436         /* Thread. */
2437         thread_ip_inc(p);
2438 }
2439
2440 static inline void
2441 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2442 {
2443         struct thread *t = &p->threads[p->thread_id];
2444         struct instruction *ip = t->ip;
2445
2446         __instr_mov_hh_exec(p, t, ip);
2447
2448         /* Thread. */
2449         thread_ip_inc(p);
2450 }
2451
2452 static inline void
2453 instr_mov_i_exec(struct rte_swx_pipeline *p)
2454 {
2455         struct thread *t = &p->threads[p->thread_id];
2456         struct instruction *ip = t->ip;
2457
2458         __instr_mov_i_exec(p, t, ip);
2459
2460         /* Thread. */
2461         thread_ip_inc(p);
2462 }
2463
2464 /*
2465  * dma.
2466  */
2467 static inline void
2468 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2469 {
2470         struct thread *t = &p->threads[p->thread_id];
2471         struct instruction *ip = t->ip;
2472
2473         __instr_dma_ht_exec(p, t, ip);
2474
2475         /* Thread. */
2476         thread_ip_inc(p);
2477 }
2478
2479 static inline void
2480 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2481 {
2482         struct thread *t = &p->threads[p->thread_id];
2483         struct instruction *ip = t->ip;
2484
2485         __instr_dma_ht2_exec(p, t, ip);
2486
2487         /* Thread. */
2488         thread_ip_inc(p);
2489 }
2490
2491 static inline void
2492 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2493 {
2494         struct thread *t = &p->threads[p->thread_id];
2495         struct instruction *ip = t->ip;
2496
2497         __instr_dma_ht3_exec(p, t, ip);
2498
2499         /* Thread. */
2500         thread_ip_inc(p);
2501 }
2502
2503 static inline void
2504 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2505 {
2506         struct thread *t = &p->threads[p->thread_id];
2507         struct instruction *ip = t->ip;
2508
2509         __instr_dma_ht4_exec(p, t, ip);
2510
2511         /* Thread. */
2512         thread_ip_inc(p);
2513 }
2514
2515 static inline void
2516 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2517 {
2518         struct thread *t = &p->threads[p->thread_id];
2519         struct instruction *ip = t->ip;
2520
2521         __instr_dma_ht5_exec(p, t, ip);
2522
2523         /* Thread. */
2524         thread_ip_inc(p);
2525 }
2526
2527 static inline void
2528 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2529 {
2530         struct thread *t = &p->threads[p->thread_id];
2531         struct instruction *ip = t->ip;
2532
2533         __instr_dma_ht6_exec(p, t, ip);
2534
2535         /* Thread. */
2536         thread_ip_inc(p);
2537 }
2538
2539 static inline void
2540 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2541 {
2542         struct thread *t = &p->threads[p->thread_id];
2543         struct instruction *ip = t->ip;
2544
2545         __instr_dma_ht7_exec(p, t, ip);
2546
2547         /* Thread. */
2548         thread_ip_inc(p);
2549 }
2550
2551 static inline void
2552 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2553 {
2554         struct thread *t = &p->threads[p->thread_id];
2555         struct instruction *ip = t->ip;
2556
2557         __instr_dma_ht8_exec(p, t, ip);
2558
2559         /* Thread. */
2560         thread_ip_inc(p);
2561 }
2562
2563 /*
2564  * alu.
2565  */
2566 static int
2567 instr_alu_add_translate(struct rte_swx_pipeline *p,
2568                         struct action *action,
2569                         char **tokens,
2570                         int n_tokens,
2571                         struct instruction *instr,
2572                         struct instruction_data *data __rte_unused)
2573 {
2574         char *dst = tokens[1], *src = tokens[2];
2575         struct field *fdst, *fsrc;
2576         uint64_t src_val;
2577         uint32_t dst_struct_id = 0, src_struct_id = 0;
2578
2579         CHECK(n_tokens == 3, EINVAL);
2580
2581         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2582         CHECK(fdst, EINVAL);
2583         CHECK(!fdst->var_size, EINVAL);
2584
2585         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2586         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2587         if (fsrc) {
2588                 CHECK(!fsrc->var_size, EINVAL);
2589
2590                 instr->type = INSTR_ALU_ADD;
2591                 if (dst[0] == 'h' && src[0] != 'h')
2592                         instr->type = INSTR_ALU_ADD_HM;
2593                 if (dst[0] != 'h' && src[0] == 'h')
2594                         instr->type = INSTR_ALU_ADD_MH;
2595                 if (dst[0] == 'h' && src[0] == 'h')
2596                         instr->type = INSTR_ALU_ADD_HH;
2597
2598                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2599                 instr->alu.dst.n_bits = fdst->n_bits;
2600                 instr->alu.dst.offset = fdst->offset / 8;
2601                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2602                 instr->alu.src.n_bits = fsrc->n_bits;
2603                 instr->alu.src.offset = fsrc->offset / 8;
2604                 return 0;
2605         }
2606
2607         /* ADD_MI, ADD_HI. */
2608         src_val = strtoull(src, &src, 0);
2609         CHECK(!src[0], EINVAL);
2610
2611         instr->type = INSTR_ALU_ADD_MI;
2612         if (dst[0] == 'h')
2613                 instr->type = INSTR_ALU_ADD_HI;
2614
2615         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2616         instr->alu.dst.n_bits = fdst->n_bits;
2617         instr->alu.dst.offset = fdst->offset / 8;
2618         instr->alu.src_val = src_val;
2619         return 0;
2620 }
2621
2622 static int
2623 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2624                         struct action *action,
2625                         char **tokens,
2626                         int n_tokens,
2627                         struct instruction *instr,
2628                         struct instruction_data *data __rte_unused)
2629 {
2630         char *dst = tokens[1], *src = tokens[2];
2631         struct field *fdst, *fsrc;
2632         uint64_t src_val;
2633         uint32_t dst_struct_id = 0, src_struct_id = 0;
2634
2635         CHECK(n_tokens == 3, EINVAL);
2636
2637         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2638         CHECK(fdst, EINVAL);
2639         CHECK(!fdst->var_size, EINVAL);
2640
2641         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2642         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2643         if (fsrc) {
2644                 CHECK(!fsrc->var_size, EINVAL);
2645
2646                 instr->type = INSTR_ALU_SUB;
2647                 if (dst[0] == 'h' && src[0] != 'h')
2648                         instr->type = INSTR_ALU_SUB_HM;
2649                 if (dst[0] != 'h' && src[0] == 'h')
2650                         instr->type = INSTR_ALU_SUB_MH;
2651                 if (dst[0] == 'h' && src[0] == 'h')
2652                         instr->type = INSTR_ALU_SUB_HH;
2653
2654                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2655                 instr->alu.dst.n_bits = fdst->n_bits;
2656                 instr->alu.dst.offset = fdst->offset / 8;
2657                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2658                 instr->alu.src.n_bits = fsrc->n_bits;
2659                 instr->alu.src.offset = fsrc->offset / 8;
2660                 return 0;
2661         }
2662
2663         /* SUB_MI, SUB_HI. */
2664         src_val = strtoull(src, &src, 0);
2665         CHECK(!src[0], EINVAL);
2666
2667         instr->type = INSTR_ALU_SUB_MI;
2668         if (dst[0] == 'h')
2669                 instr->type = INSTR_ALU_SUB_HI;
2670
2671         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2672         instr->alu.dst.n_bits = fdst->n_bits;
2673         instr->alu.dst.offset = fdst->offset / 8;
2674         instr->alu.src_val = src_val;
2675         return 0;
2676 }
2677
2678 static int
2679 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2680                           struct action *action __rte_unused,
2681                           char **tokens,
2682                           int n_tokens,
2683                           struct instruction *instr,
2684                           struct instruction_data *data __rte_unused)
2685 {
2686         char *dst = tokens[1], *src = tokens[2];
2687         struct header *hdst, *hsrc;
2688         struct field *fdst, *fsrc;
2689
2690         CHECK(n_tokens == 3, EINVAL);
2691
2692         fdst = header_field_parse(p, dst, &hdst);
2693         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2694         CHECK(!fdst->var_size, EINVAL);
2695
2696         /* CKADD_FIELD. */
2697         fsrc = header_field_parse(p, src, &hsrc);
2698         if (fsrc) {
2699                 CHECK(!fsrc->var_size, EINVAL);
2700
2701                 instr->type = INSTR_ALU_CKADD_FIELD;
2702                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2703                 instr->alu.dst.n_bits = fdst->n_bits;
2704                 instr->alu.dst.offset = fdst->offset / 8;
2705                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2706                 instr->alu.src.n_bits = fsrc->n_bits;
2707                 instr->alu.src.offset = fsrc->offset / 8;
2708                 return 0;
2709         }
2710
2711         /* CKADD_STRUCT, CKADD_STRUCT20. */
2712         hsrc = header_parse(p, src);
2713         CHECK(hsrc, EINVAL);
2714         CHECK(!hsrc->st->var_size, EINVAL);
2715
2716         instr->type = INSTR_ALU_CKADD_STRUCT;
2717         if ((hsrc->st->n_bits / 8) == 20)
2718                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2719
2720         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2721         instr->alu.dst.n_bits = fdst->n_bits;
2722         instr->alu.dst.offset = fdst->offset / 8;
2723         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2724         instr->alu.src.n_bits = hsrc->st->n_bits;
2725         instr->alu.src.offset = 0; /* Unused. */
2726         return 0;
2727 }
2728
2729 static int
2730 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2731                           struct action *action __rte_unused,
2732                           char **tokens,
2733                           int n_tokens,
2734                           struct instruction *instr,
2735                           struct instruction_data *data __rte_unused)
2736 {
2737         char *dst = tokens[1], *src = tokens[2];
2738         struct header *hdst, *hsrc;
2739         struct field *fdst, *fsrc;
2740
2741         CHECK(n_tokens == 3, EINVAL);
2742
2743         fdst = header_field_parse(p, dst, &hdst);
2744         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2745         CHECK(!fdst->var_size, EINVAL);
2746
2747         fsrc = header_field_parse(p, src, &hsrc);
2748         CHECK(fsrc, EINVAL);
2749         CHECK(!fsrc->var_size, EINVAL);
2750
2751         instr->type = INSTR_ALU_CKSUB_FIELD;
2752         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2753         instr->alu.dst.n_bits = fdst->n_bits;
2754         instr->alu.dst.offset = fdst->offset / 8;
2755         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2756         instr->alu.src.n_bits = fsrc->n_bits;
2757         instr->alu.src.offset = fsrc->offset / 8;
2758         return 0;
2759 }
2760
2761 static int
2762 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2763                         struct action *action,
2764                         char **tokens,
2765                         int n_tokens,
2766                         struct instruction *instr,
2767                         struct instruction_data *data __rte_unused)
2768 {
2769         char *dst = tokens[1], *src = tokens[2];
2770         struct field *fdst, *fsrc;
2771         uint64_t src_val;
2772         uint32_t dst_struct_id = 0, src_struct_id = 0;
2773
2774         CHECK(n_tokens == 3, EINVAL);
2775
2776         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2777         CHECK(fdst, EINVAL);
2778         CHECK(!fdst->var_size, EINVAL);
2779
2780         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2781         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2782         if (fsrc) {
2783                 CHECK(!fsrc->var_size, EINVAL);
2784
2785                 instr->type = INSTR_ALU_SHL;
2786                 if (dst[0] == 'h' && src[0] != 'h')
2787                         instr->type = INSTR_ALU_SHL_HM;
2788                 if (dst[0] != 'h' && src[0] == 'h')
2789                         instr->type = INSTR_ALU_SHL_MH;
2790                 if (dst[0] == 'h' && src[0] == 'h')
2791                         instr->type = INSTR_ALU_SHL_HH;
2792
2793                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2794                 instr->alu.dst.n_bits = fdst->n_bits;
2795                 instr->alu.dst.offset = fdst->offset / 8;
2796                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2797                 instr->alu.src.n_bits = fsrc->n_bits;
2798                 instr->alu.src.offset = fsrc->offset / 8;
2799                 return 0;
2800         }
2801
2802         /* SHL_MI, SHL_HI. */
2803         src_val = strtoull(src, &src, 0);
2804         CHECK(!src[0], EINVAL);
2805
2806         instr->type = INSTR_ALU_SHL_MI;
2807         if (dst[0] == 'h')
2808                 instr->type = INSTR_ALU_SHL_HI;
2809
2810         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2811         instr->alu.dst.n_bits = fdst->n_bits;
2812         instr->alu.dst.offset = fdst->offset / 8;
2813         instr->alu.src_val = src_val;
2814         return 0;
2815 }
2816
2817 static int
2818 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2819                         struct action *action,
2820                         char **tokens,
2821                         int n_tokens,
2822                         struct instruction *instr,
2823                         struct instruction_data *data __rte_unused)
2824 {
2825         char *dst = tokens[1], *src = tokens[2];
2826         struct field *fdst, *fsrc;
2827         uint64_t src_val;
2828         uint32_t dst_struct_id = 0, src_struct_id = 0;
2829
2830         CHECK(n_tokens == 3, EINVAL);
2831
2832         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2833         CHECK(fdst, EINVAL);
2834         CHECK(!fdst->var_size, EINVAL);
2835
2836         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2837         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2838         if (fsrc) {
2839                 CHECK(!fsrc->var_size, EINVAL);
2840
2841                 instr->type = INSTR_ALU_SHR;
2842                 if (dst[0] == 'h' && src[0] != 'h')
2843                         instr->type = INSTR_ALU_SHR_HM;
2844                 if (dst[0] != 'h' && src[0] == 'h')
2845                         instr->type = INSTR_ALU_SHR_MH;
2846                 if (dst[0] == 'h' && src[0] == 'h')
2847                         instr->type = INSTR_ALU_SHR_HH;
2848
2849                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2850                 instr->alu.dst.n_bits = fdst->n_bits;
2851                 instr->alu.dst.offset = fdst->offset / 8;
2852                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2853                 instr->alu.src.n_bits = fsrc->n_bits;
2854                 instr->alu.src.offset = fsrc->offset / 8;
2855                 return 0;
2856         }
2857
2858         /* SHR_MI, SHR_HI. */
2859         src_val = strtoull(src, &src, 0);
2860         CHECK(!src[0], EINVAL);
2861
2862         instr->type = INSTR_ALU_SHR_MI;
2863         if (dst[0] == 'h')
2864                 instr->type = INSTR_ALU_SHR_HI;
2865
2866         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2867         instr->alu.dst.n_bits = fdst->n_bits;
2868         instr->alu.dst.offset = fdst->offset / 8;
2869         instr->alu.src_val = src_val;
2870         return 0;
2871 }
2872
2873 static int
2874 instr_alu_and_translate(struct rte_swx_pipeline *p,
2875                         struct action *action,
2876                         char **tokens,
2877                         int n_tokens,
2878                         struct instruction *instr,
2879                         struct instruction_data *data __rte_unused)
2880 {
2881         char *dst = tokens[1], *src = tokens[2];
2882         struct field *fdst, *fsrc;
2883         uint64_t src_val;
2884         uint32_t dst_struct_id = 0, src_struct_id = 0;
2885
2886         CHECK(n_tokens == 3, EINVAL);
2887
2888         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2889         CHECK(fdst, EINVAL);
2890         CHECK(!fdst->var_size, EINVAL);
2891
2892         /* AND, AND_MH, AND_HM, AND_HH. */
2893         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2894         if (fsrc) {
2895                 CHECK(!fsrc->var_size, EINVAL);
2896
2897                 instr->type = INSTR_ALU_AND;
2898                 if (dst[0] != 'h' && src[0] == 'h')
2899                         instr->type = INSTR_ALU_AND_MH;
2900                 if (dst[0] == 'h' && src[0] != 'h')
2901                         instr->type = INSTR_ALU_AND_HM;
2902                 if (dst[0] == 'h' && src[0] == 'h')
2903                         instr->type = INSTR_ALU_AND_HH;
2904
2905                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2906                 instr->alu.dst.n_bits = fdst->n_bits;
2907                 instr->alu.dst.offset = fdst->offset / 8;
2908                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2909                 instr->alu.src.n_bits = fsrc->n_bits;
2910                 instr->alu.src.offset = fsrc->offset / 8;
2911                 return 0;
2912         }
2913
2914         /* AND_I. */
2915         src_val = strtoull(src, &src, 0);
2916         CHECK(!src[0], EINVAL);
2917
2918         if (dst[0] == 'h')
2919                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2920
2921         instr->type = INSTR_ALU_AND_I;
2922         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2923         instr->alu.dst.n_bits = fdst->n_bits;
2924         instr->alu.dst.offset = fdst->offset / 8;
2925         instr->alu.src_val = src_val;
2926         return 0;
2927 }
2928
2929 static int
2930 instr_alu_or_translate(struct rte_swx_pipeline *p,
2931                        struct action *action,
2932                        char **tokens,
2933                        int n_tokens,
2934                        struct instruction *instr,
2935                        struct instruction_data *data __rte_unused)
2936 {
2937         char *dst = tokens[1], *src = tokens[2];
2938         struct field *fdst, *fsrc;
2939         uint64_t src_val;
2940         uint32_t dst_struct_id = 0, src_struct_id = 0;
2941
2942         CHECK(n_tokens == 3, EINVAL);
2943
2944         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2945         CHECK(fdst, EINVAL);
2946         CHECK(!fdst->var_size, EINVAL);
2947
2948         /* OR, OR_MH, OR_HM, OR_HH. */
2949         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2950         if (fsrc) {
2951                 CHECK(!fsrc->var_size, EINVAL);
2952
2953                 instr->type = INSTR_ALU_OR;
2954                 if (dst[0] != 'h' && src[0] == 'h')
2955                         instr->type = INSTR_ALU_OR_MH;
2956                 if (dst[0] == 'h' && src[0] != 'h')
2957                         instr->type = INSTR_ALU_OR_HM;
2958                 if (dst[0] == 'h' && src[0] == 'h')
2959                         instr->type = INSTR_ALU_OR_HH;
2960
2961                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2962                 instr->alu.dst.n_bits = fdst->n_bits;
2963                 instr->alu.dst.offset = fdst->offset / 8;
2964                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2965                 instr->alu.src.n_bits = fsrc->n_bits;
2966                 instr->alu.src.offset = fsrc->offset / 8;
2967                 return 0;
2968         }
2969
2970         /* OR_I. */
2971         src_val = strtoull(src, &src, 0);
2972         CHECK(!src[0], EINVAL);
2973
2974         if (dst[0] == 'h')
2975                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2976
2977         instr->type = INSTR_ALU_OR_I;
2978         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2979         instr->alu.dst.n_bits = fdst->n_bits;
2980         instr->alu.dst.offset = fdst->offset / 8;
2981         instr->alu.src_val = src_val;
2982         return 0;
2983 }
2984
2985 static int
2986 instr_alu_xor_translate(struct rte_swx_pipeline *p,
2987                         struct action *action,
2988                         char **tokens,
2989                         int n_tokens,
2990                         struct instruction *instr,
2991                         struct instruction_data *data __rte_unused)
2992 {
2993         char *dst = tokens[1], *src = tokens[2];
2994         struct field *fdst, *fsrc;
2995         uint64_t src_val;
2996         uint32_t dst_struct_id = 0, src_struct_id = 0;
2997
2998         CHECK(n_tokens == 3, EINVAL);
2999
3000         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3001         CHECK(fdst, EINVAL);
3002         CHECK(!fdst->var_size, EINVAL);
3003
3004         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3006         if (fsrc) {
3007                 CHECK(!fsrc->var_size, EINVAL);
3008
3009                 instr->type = INSTR_ALU_XOR;
3010                 if (dst[0] != 'h' && src[0] == 'h')
3011                         instr->type = INSTR_ALU_XOR_MH;
3012                 if (dst[0] == 'h' && src[0] != 'h')
3013                         instr->type = INSTR_ALU_XOR_HM;
3014                 if (dst[0] == 'h' && src[0] == 'h')
3015                         instr->type = INSTR_ALU_XOR_HH;
3016
3017                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3018                 instr->alu.dst.n_bits = fdst->n_bits;
3019                 instr->alu.dst.offset = fdst->offset / 8;
3020                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3021                 instr->alu.src.n_bits = fsrc->n_bits;
3022                 instr->alu.src.offset = fsrc->offset / 8;
3023                 return 0;
3024         }
3025
3026         /* XOR_I. */
3027         src_val = strtoull(src, &src, 0);
3028         CHECK(!src[0], EINVAL);
3029
3030         if (dst[0] == 'h')
3031                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3032
3033         instr->type = INSTR_ALU_XOR_I;
3034         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3035         instr->alu.dst.n_bits = fdst->n_bits;
3036         instr->alu.dst.offset = fdst->offset / 8;
3037         instr->alu.src_val = src_val;
3038         return 0;
3039 }
3040
3041 static inline void
3042 instr_alu_add_exec(struct rte_swx_pipeline *p)
3043 {
3044         struct thread *t = &p->threads[p->thread_id];
3045         struct instruction *ip = t->ip;
3046
3047         /* Structs */
3048         __instr_alu_add_exec(p, t, ip);
3049
3050         /* Thread. */
3051         thread_ip_inc(p);
3052 }
3053
3054 static inline void
3055 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3056 {
3057         struct thread *t = &p->threads[p->thread_id];
3058         struct instruction *ip = t->ip;
3059
3060         /* Structs. */
3061         __instr_alu_add_mh_exec(p, t, ip);
3062
3063         /* Thread. */
3064         thread_ip_inc(p);
3065 }
3066
3067 static inline void
3068 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3069 {
3070         struct thread *t = &p->threads[p->thread_id];
3071         struct instruction *ip = t->ip;
3072
3073         /* Structs. */
3074         __instr_alu_add_hm_exec(p, t, ip);
3075
3076         /* Thread. */
3077         thread_ip_inc(p);
3078 }
3079
3080 static inline void
3081 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3082 {
3083         struct thread *t = &p->threads[p->thread_id];
3084         struct instruction *ip = t->ip;
3085
3086         /* Structs. */
3087         __instr_alu_add_hh_exec(p, t, ip);
3088
3089         /* Thread. */
3090         thread_ip_inc(p);
3091 }
3092
3093 static inline void
3094 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3095 {
3096         struct thread *t = &p->threads[p->thread_id];
3097         struct instruction *ip = t->ip;
3098
3099         /* Structs. */
3100         __instr_alu_add_mi_exec(p, t, ip);
3101
3102         /* Thread. */
3103         thread_ip_inc(p);
3104 }
3105
3106 static inline void
3107 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3108 {
3109         struct thread *t = &p->threads[p->thread_id];
3110         struct instruction *ip = t->ip;
3111
3112         /* Structs. */
3113         __instr_alu_add_hi_exec(p, t, ip);
3114
3115         /* Thread. */
3116         thread_ip_inc(p);
3117 }
3118
3119 static inline void
3120 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3121 {
3122         struct thread *t = &p->threads[p->thread_id];
3123         struct instruction *ip = t->ip;
3124
3125         /* Structs. */
3126         __instr_alu_sub_exec(p, t, ip);
3127
3128         /* Thread. */
3129         thread_ip_inc(p);
3130 }
3131
3132 static inline void
3133 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3134 {
3135         struct thread *t = &p->threads[p->thread_id];
3136         struct instruction *ip = t->ip;
3137
3138         /* Structs. */
3139         __instr_alu_sub_mh_exec(p, t, ip);
3140
3141         /* Thread. */
3142         thread_ip_inc(p);
3143 }
3144
3145 static inline void
3146 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3147 {
3148         struct thread *t = &p->threads[p->thread_id];
3149         struct instruction *ip = t->ip;
3150
3151         /* Structs. */
3152         __instr_alu_sub_hm_exec(p, t, ip);
3153
3154         /* Thread. */
3155         thread_ip_inc(p);
3156 }
3157
3158 static inline void
3159 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3160 {
3161         struct thread *t = &p->threads[p->thread_id];
3162         struct instruction *ip = t->ip;
3163
3164         /* Structs. */
3165         __instr_alu_sub_hh_exec(p, t, ip);
3166
3167         /* Thread. */
3168         thread_ip_inc(p);
3169 }
3170
3171 static inline void
3172 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3173 {
3174         struct thread *t = &p->threads[p->thread_id];
3175         struct instruction *ip = t->ip;
3176
3177         /* Structs. */
3178         __instr_alu_sub_mi_exec(p, t, ip);
3179
3180         /* Thread. */
3181         thread_ip_inc(p);
3182 }
3183
3184 static inline void
3185 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3186 {
3187         struct thread *t = &p->threads[p->thread_id];
3188         struct instruction *ip = t->ip;
3189
3190         /* Structs. */
3191         __instr_alu_sub_hi_exec(p, t, ip);
3192
3193         /* Thread. */
3194         thread_ip_inc(p);
3195 }
3196
3197 static inline void
3198 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3199 {
3200         struct thread *t = &p->threads[p->thread_id];
3201         struct instruction *ip = t->ip;
3202
3203         /* Structs. */
3204         __instr_alu_shl_exec(p, t, ip);
3205
3206         /* Thread. */
3207         thread_ip_inc(p);
3208 }
3209
3210 static inline void
3211 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3212 {
3213         struct thread *t = &p->threads[p->thread_id];
3214         struct instruction *ip = t->ip;
3215
3216         /* Structs. */
3217         __instr_alu_shl_mh_exec(p, t, ip);
3218
3219         /* Thread. */
3220         thread_ip_inc(p);
3221 }
3222
3223 static inline void
3224 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3225 {
3226         struct thread *t = &p->threads[p->thread_id];
3227         struct instruction *ip = t->ip;
3228
3229         /* Structs. */
3230         __instr_alu_shl_hm_exec(p, t, ip);
3231
3232         /* Thread. */
3233         thread_ip_inc(p);
3234 }
3235
3236 static inline void
3237 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3238 {
3239         struct thread *t = &p->threads[p->thread_id];
3240         struct instruction *ip = t->ip;
3241
3242         /* Structs. */
3243         __instr_alu_shl_hh_exec(p, t, ip);
3244
3245         /* Thread. */
3246         thread_ip_inc(p);
3247 }
3248
3249 static inline void
3250 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3251 {
3252         struct thread *t = &p->threads[p->thread_id];
3253         struct instruction *ip = t->ip;
3254
3255         /* Structs. */
3256         __instr_alu_shl_mi_exec(p, t, ip);
3257
3258         /* Thread. */
3259         thread_ip_inc(p);
3260 }
3261
3262 static inline void
3263 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3264 {
3265         struct thread *t = &p->threads[p->thread_id];
3266         struct instruction *ip = t->ip;
3267
3268         /* Structs. */
3269         __instr_alu_shl_hi_exec(p, t, ip);
3270
3271         /* Thread. */
3272         thread_ip_inc(p);
3273 }
3274
3275 static inline void
3276 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3277 {
3278         struct thread *t = &p->threads[p->thread_id];
3279         struct instruction *ip = t->ip;
3280
3281         /* Structs. */
3282         __instr_alu_shr_exec(p, t, ip);
3283
3284         /* Thread. */
3285         thread_ip_inc(p);
3286 }
3287
3288 static inline void
3289 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3290 {
3291         struct thread *t = &p->threads[p->thread_id];
3292         struct instruction *ip = t->ip;
3293
3294         /* Structs. */
3295         __instr_alu_shr_mh_exec(p, t, ip);
3296
3297         /* Thread. */
3298         thread_ip_inc(p);
3299 }
3300
3301 static inline void
3302 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3303 {
3304         struct thread *t = &p->threads[p->thread_id];
3305         struct instruction *ip = t->ip;
3306
3307         /* Structs. */
3308         __instr_alu_shr_hm_exec(p, t, ip);
3309
3310         /* Thread. */
3311         thread_ip_inc(p);
3312 }
3313
3314 static inline void
3315 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3316 {
3317         struct thread *t = &p->threads[p->thread_id];
3318         struct instruction *ip = t->ip;
3319
3320         /* Structs. */
3321         __instr_alu_shr_hh_exec(p, t, ip);
3322
3323         /* Thread. */
3324         thread_ip_inc(p);
3325 }
3326
3327 static inline void
3328 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3329 {
3330         struct thread *t = &p->threads[p->thread_id];
3331         struct instruction *ip = t->ip;
3332
3333         /* Structs. */
3334         __instr_alu_shr_mi_exec(p, t, ip);
3335
3336         /* Thread. */
3337         thread_ip_inc(p);
3338 }
3339
3340 static inline void
3341 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3342 {
3343         struct thread *t = &p->threads[p->thread_id];
3344         struct instruction *ip = t->ip;
3345
3346         /* Structs. */
3347         __instr_alu_shr_hi_exec(p, t, ip);
3348
3349         /* Thread. */
3350         thread_ip_inc(p);
3351 }
3352
3353 static inline void
3354 instr_alu_and_exec(struct rte_swx_pipeline *p)
3355 {
3356         struct thread *t = &p->threads[p->thread_id];
3357         struct instruction *ip = t->ip;
3358
3359         /* Structs. */
3360         __instr_alu_and_exec(p, t, ip);
3361
3362         /* Thread. */
3363         thread_ip_inc(p);
3364 }
3365
3366 static inline void
3367 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3368 {
3369         struct thread *t = &p->threads[p->thread_id];
3370         struct instruction *ip = t->ip;
3371
3372         /* Structs. */
3373         __instr_alu_and_mh_exec(p, t, ip);
3374
3375         /* Thread. */
3376         thread_ip_inc(p);
3377 }
3378
3379 static inline void
3380 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3381 {
3382         struct thread *t = &p->threads[p->thread_id];
3383         struct instruction *ip = t->ip;
3384
3385         /* Structs. */
3386         __instr_alu_and_hm_exec(p, t, ip);
3387
3388         /* Thread. */
3389         thread_ip_inc(p);
3390 }
3391
3392 static inline void
3393 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3394 {
3395         struct thread *t = &p->threads[p->thread_id];
3396         struct instruction *ip = t->ip;
3397
3398         /* Structs. */
3399         __instr_alu_and_hh_exec(p, t, ip);
3400
3401         /* Thread. */
3402         thread_ip_inc(p);
3403 }
3404
3405 static inline void
3406 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3407 {
3408         struct thread *t = &p->threads[p->thread_id];
3409         struct instruction *ip = t->ip;
3410
3411         /* Structs. */
3412         __instr_alu_and_i_exec(p, t, ip);
3413
3414         /* Thread. */
3415         thread_ip_inc(p);
3416 }
3417
3418 static inline void
3419 instr_alu_or_exec(struct rte_swx_pipeline *p)
3420 {
3421         struct thread *t = &p->threads[p->thread_id];
3422         struct instruction *ip = t->ip;
3423
3424         /* Structs. */
3425         __instr_alu_or_exec(p, t, ip);
3426
3427         /* Thread. */
3428         thread_ip_inc(p);
3429 }
3430
3431 static inline void
3432 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3433 {
3434         struct thread *t = &p->threads[p->thread_id];
3435         struct instruction *ip = t->ip;
3436
3437         /* Structs. */
3438         __instr_alu_or_mh_exec(p, t, ip);
3439
3440         /* Thread. */
3441         thread_ip_inc(p);
3442 }
3443
3444 static inline void
3445 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3446 {
3447         struct thread *t = &p->threads[p->thread_id];
3448         struct instruction *ip = t->ip;
3449
3450         /* Structs. */
3451         __instr_alu_or_hm_exec(p, t, ip);
3452
3453         /* Thread. */
3454         thread_ip_inc(p);
3455 }
3456
3457 static inline void
3458 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3459 {
3460         struct thread *t = &p->threads[p->thread_id];
3461         struct instruction *ip = t->ip;
3462
3463         /* Structs. */
3464         __instr_alu_or_hh_exec(p, t, ip);
3465
3466         /* Thread. */
3467         thread_ip_inc(p);
3468 }
3469
3470 static inline void
3471 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3472 {
3473         struct thread *t = &p->threads[p->thread_id];
3474         struct instruction *ip = t->ip;
3475
3476         /* Structs. */
3477         __instr_alu_or_i_exec(p, t, ip);
3478
3479         /* Thread. */
3480         thread_ip_inc(p);
3481 }
3482
3483 static inline void
3484 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3485 {
3486         struct thread *t = &p->threads[p->thread_id];
3487         struct instruction *ip = t->ip;
3488
3489         /* Structs. */
3490         __instr_alu_xor_exec(p, t, ip);
3491
3492         /* Thread. */
3493         thread_ip_inc(p);
3494 }
3495
3496 static inline void
3497 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3498 {
3499         struct thread *t = &p->threads[p->thread_id];
3500         struct instruction *ip = t->ip;
3501
3502         /* Structs. */
3503         __instr_alu_xor_mh_exec(p, t, ip);
3504
3505         /* Thread. */
3506         thread_ip_inc(p);
3507 }
3508
3509 static inline void
3510 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3511 {
3512         struct thread *t = &p->threads[p->thread_id];
3513         struct instruction *ip = t->ip;
3514
3515         /* Structs. */
3516         __instr_alu_xor_hm_exec(p, t, ip);
3517
3518         /* Thread. */
3519         thread_ip_inc(p);
3520 }
3521
3522 static inline void
3523 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3524 {
3525         struct thread *t = &p->threads[p->thread_id];
3526         struct instruction *ip = t->ip;
3527
3528         /* Structs. */
3529         __instr_alu_xor_hh_exec(p, t, ip);
3530
3531         /* Thread. */
3532         thread_ip_inc(p);
3533 }
3534
3535 static inline void
3536 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3537 {
3538         struct thread *t = &p->threads[p->thread_id];
3539         struct instruction *ip = t->ip;
3540
3541         /* Structs. */
3542         __instr_alu_xor_i_exec(p, t, ip);
3543
3544         /* Thread. */
3545         thread_ip_inc(p);
3546 }
3547
3548 static inline void
3549 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3550 {
3551         struct thread *t = &p->threads[p->thread_id];
3552         struct instruction *ip = t->ip;
3553
3554         /* Structs. */
3555         __instr_alu_ckadd_field_exec(p, t, ip);
3556
3557         /* Thread. */
3558         thread_ip_inc(p);
3559 }
3560
3561 static inline void
3562 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3563 {
3564         struct thread *t = &p->threads[p->thread_id];
3565         struct instruction *ip = t->ip;
3566
3567         /* Structs. */
3568         __instr_alu_cksub_field_exec(p, t, ip);
3569
3570         /* Thread. */
3571         thread_ip_inc(p);
3572 }
3573
3574 static inline void
3575 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3576 {
3577         struct thread *t = &p->threads[p->thread_id];
3578         struct instruction *ip = t->ip;
3579
3580         /* Structs. */
3581         __instr_alu_ckadd_struct20_exec(p, t, ip);
3582
3583         /* Thread. */
3584         thread_ip_inc(p);
3585 }
3586
3587 static inline void
3588 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3589 {
3590         struct thread *t = &p->threads[p->thread_id];
3591         struct instruction *ip = t->ip;
3592
3593         /* Structs. */
3594         __instr_alu_ckadd_struct_exec(p, t, ip);
3595
3596         /* Thread. */
3597         thread_ip_inc(p);
3598 }
3599
3600 /*
3601  * Register array.
3602  */
3603 static struct regarray *
3604 regarray_find(struct rte_swx_pipeline *p, const char *name);
3605
3606 static int
3607 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3608                       struct action *action,
3609                       char **tokens,
3610                       int n_tokens,
3611                       struct instruction *instr,
3612                       struct instruction_data *data __rte_unused)
3613 {
3614         char *regarray = tokens[1], *idx = tokens[2];
3615         struct regarray *r;
3616         struct field *fidx;
3617         uint32_t idx_struct_id, idx_val;
3618
3619         CHECK(n_tokens == 3, EINVAL);
3620
3621         r = regarray_find(p, regarray);
3622         CHECK(r, EINVAL);
3623
3624         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3625         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3626         if (fidx) {
3627                 CHECK(!fidx->var_size, EINVAL);
3628
3629                 instr->type = INSTR_REGPREFETCH_RM;
3630                 if (idx[0] == 'h')
3631                         instr->type = INSTR_REGPREFETCH_RH;
3632
3633                 instr->regarray.regarray_id = r->id;
3634                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3635                 instr->regarray.idx.n_bits = fidx->n_bits;
3636                 instr->regarray.idx.offset = fidx->offset / 8;
3637                 instr->regarray.dstsrc_val = 0; /* Unused. */
3638                 return 0;
3639         }
3640
3641         /* REGPREFETCH_RI. */
3642         idx_val = strtoul(idx, &idx, 0);
3643         CHECK(!idx[0], EINVAL);
3644
3645         instr->type = INSTR_REGPREFETCH_RI;
3646         instr->regarray.regarray_id = r->id;
3647         instr->regarray.idx_val = idx_val;
3648         instr->regarray.dstsrc_val = 0; /* Unused. */
3649         return 0;
3650 }
3651
3652 static int
3653 instr_regrd_translate(struct rte_swx_pipeline *p,
3654                       struct action *action,
3655                       char **tokens,
3656                       int n_tokens,
3657                       struct instruction *instr,
3658                       struct instruction_data *data __rte_unused)
3659 {
3660         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3661         struct regarray *r;
3662         struct field *fdst, *fidx;
3663         uint32_t dst_struct_id, idx_struct_id, idx_val;
3664
3665         CHECK(n_tokens == 4, EINVAL);
3666
3667         r = regarray_find(p, regarray);
3668         CHECK(r, EINVAL);
3669
3670         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3671         CHECK(fdst, EINVAL);
3672         CHECK(!fdst->var_size, EINVAL);
3673
3674         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3675         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3676         if (fidx) {
3677                 CHECK(!fidx->var_size, EINVAL);
3678
3679                 instr->type = INSTR_REGRD_MRM;
3680                 if (dst[0] == 'h' && idx[0] != 'h')
3681                         instr->type = INSTR_REGRD_HRM;
3682                 if (dst[0] != 'h' && idx[0] == 'h')
3683                         instr->type = INSTR_REGRD_MRH;
3684                 if (dst[0] == 'h' && idx[0] == 'h')
3685                         instr->type = INSTR_REGRD_HRH;
3686
3687                 instr->regarray.regarray_id = r->id;
3688                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3689                 instr->regarray.idx.n_bits = fidx->n_bits;
3690                 instr->regarray.idx.offset = fidx->offset / 8;
3691                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3692                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3693                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3694                 return 0;
3695         }
3696
3697         /* REGRD_MRI, REGRD_HRI. */
3698         idx_val = strtoul(idx, &idx, 0);
3699         CHECK(!idx[0], EINVAL);
3700
3701         instr->type = INSTR_REGRD_MRI;
3702         if (dst[0] == 'h')
3703                 instr->type = INSTR_REGRD_HRI;
3704
3705         instr->regarray.regarray_id = r->id;
3706         instr->regarray.idx_val = idx_val;
3707         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3708         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3709         instr->regarray.dstsrc.offset = fdst->offset / 8;
3710         return 0;
3711 }
3712
3713 static int
3714 instr_regwr_translate(struct rte_swx_pipeline *p,
3715                       struct action *action,
3716                       char **tokens,
3717                       int n_tokens,
3718                       struct instruction *instr,
3719                       struct instruction_data *data __rte_unused)
3720 {
3721         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3722         struct regarray *r;
3723         struct field *fidx, *fsrc;
3724         uint64_t src_val;
3725         uint32_t idx_struct_id, idx_val, src_struct_id;
3726
3727         CHECK(n_tokens == 4, EINVAL);
3728
3729         r = regarray_find(p, regarray);
3730         CHECK(r, EINVAL);
3731
3732         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3733         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3734         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3735         if (fidx && fsrc) {
3736                 CHECK(!fidx->var_size, EINVAL);
3737                 CHECK(!fsrc->var_size, EINVAL);
3738
3739                 instr->type = INSTR_REGWR_RMM;
3740                 if (idx[0] == 'h' && src[0] != 'h')
3741                         instr->type = INSTR_REGWR_RHM;
3742                 if (idx[0] != 'h' && src[0] == 'h')
3743                         instr->type = INSTR_REGWR_RMH;
3744                 if (idx[0] == 'h' && src[0] == 'h')
3745                         instr->type = INSTR_REGWR_RHH;
3746
3747                 instr->regarray.regarray_id = r->id;
3748                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3749                 instr->regarray.idx.n_bits = fidx->n_bits;
3750                 instr->regarray.idx.offset = fidx->offset / 8;
3751                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3752                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3753                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3754                 return 0;
3755         }
3756
3757         /* REGWR_RHI, REGWR_RMI. */
3758         if (fidx && !fsrc) {
3759                 CHECK(!fidx->var_size, EINVAL);
3760
3761                 src_val = strtoull(src, &src, 0);
3762                 CHECK(!src[0], EINVAL);
3763
3764                 instr->type = INSTR_REGWR_RMI;
3765                 if (idx[0] == 'h')
3766                         instr->type = INSTR_REGWR_RHI;
3767
3768                 instr->regarray.regarray_id = r->id;
3769                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3770                 instr->regarray.idx.n_bits = fidx->n_bits;
3771                 instr->regarray.idx.offset = fidx->offset / 8;
3772                 instr->regarray.dstsrc_val = src_val;
3773                 return 0;
3774         }
3775
3776         /* REGWR_RIH, REGWR_RIM. */
3777         if (!fidx && fsrc) {
3778                 idx_val = strtoul(idx, &idx, 0);
3779                 CHECK(!idx[0], EINVAL);
3780
3781                 CHECK(!fsrc->var_size, EINVAL);
3782
3783                 instr->type = INSTR_REGWR_RIM;
3784                 if (src[0] == 'h')
3785                         instr->type = INSTR_REGWR_RIH;
3786
3787                 instr->regarray.regarray_id = r->id;
3788                 instr->regarray.idx_val = idx_val;
3789                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3790                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3791                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3792                 return 0;
3793         }
3794
3795         /* REGWR_RII. */
3796         src_val = strtoull(src, &src, 0);
3797         CHECK(!src[0], EINVAL);
3798
3799         idx_val = strtoul(idx, &idx, 0);
3800         CHECK(!idx[0], EINVAL);
3801
3802         instr->type = INSTR_REGWR_RII;
3803         instr->regarray.idx_val = idx_val;
3804         instr->regarray.dstsrc_val = src_val;
3805
3806         return 0;
3807 }
3808
3809 static int
3810 instr_regadd_translate(struct rte_swx_pipeline *p,
3811                        struct action *action,
3812                        char **tokens,
3813                        int n_tokens,
3814                        struct instruction *instr,
3815                        struct instruction_data *data __rte_unused)
3816 {
3817         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3818         struct regarray *r;
3819         struct field *fidx, *fsrc;
3820         uint64_t src_val;
3821         uint32_t idx_struct_id, idx_val, src_struct_id;
3822
3823         CHECK(n_tokens == 4, EINVAL);
3824
3825         r = regarray_find(p, regarray);
3826         CHECK(r, EINVAL);
3827
3828         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3829         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3830         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3831         if (fidx && fsrc) {
3832                 CHECK(!fidx->var_size, EINVAL);
3833                 CHECK(!fsrc->var_size, EINVAL);
3834
3835                 instr->type = INSTR_REGADD_RMM;
3836                 if (idx[0] == 'h' && src[0] != 'h')
3837                         instr->type = INSTR_REGADD_RHM;
3838                 if (idx[0] != 'h' && src[0] == 'h')
3839                         instr->type = INSTR_REGADD_RMH;
3840                 if (idx[0] == 'h' && src[0] == 'h')
3841                         instr->type = INSTR_REGADD_RHH;
3842
3843                 instr->regarray.regarray_id = r->id;
3844                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3845                 instr->regarray.idx.n_bits = fidx->n_bits;
3846                 instr->regarray.idx.offset = fidx->offset / 8;
3847                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3848                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3849                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3850                 return 0;
3851         }
3852
3853         /* REGADD_RHI, REGADD_RMI. */
3854         if (fidx && !fsrc) {
3855                 CHECK(!fidx->var_size, EINVAL);
3856
3857                 src_val = strtoull(src, &src, 0);
3858                 CHECK(!src[0], EINVAL);
3859
3860                 instr->type = INSTR_REGADD_RMI;
3861                 if (idx[0] == 'h')
3862                         instr->type = INSTR_REGADD_RHI;
3863
3864                 instr->regarray.regarray_id = r->id;
3865                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3866                 instr->regarray.idx.n_bits = fidx->n_bits;
3867                 instr->regarray.idx.offset = fidx->offset / 8;
3868                 instr->regarray.dstsrc_val = src_val;
3869                 return 0;
3870         }
3871
3872         /* REGADD_RIH, REGADD_RIM. */
3873         if (!fidx && fsrc) {
3874                 idx_val = strtoul(idx, &idx, 0);
3875                 CHECK(!idx[0], EINVAL);
3876
3877                 CHECK(!fsrc->var_size, EINVAL);
3878
3879                 instr->type = INSTR_REGADD_RIM;
3880                 if (src[0] == 'h')
3881                         instr->type = INSTR_REGADD_RIH;
3882
3883                 instr->regarray.regarray_id = r->id;
3884                 instr->regarray.idx_val = idx_val;
3885                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3886                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3887                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3888                 return 0;
3889         }
3890
3891         /* REGADD_RII. */
3892         src_val = strtoull(src, &src, 0);
3893         CHECK(!src[0], EINVAL);
3894
3895         idx_val = strtoul(idx, &idx, 0);
3896         CHECK(!idx[0], EINVAL);
3897
3898         instr->type = INSTR_REGADD_RII;
3899         instr->regarray.idx_val = idx_val;
3900         instr->regarray.dstsrc_val = src_val;
3901         return 0;
3902 }
3903
3904 static inline void
3905 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
3906 {
3907         struct thread *t = &p->threads[p->thread_id];
3908         struct instruction *ip = t->ip;
3909
3910         /* Structs. */
3911         __instr_regprefetch_rh_exec(p, t, ip);
3912
3913         /* Thread. */
3914         thread_ip_inc(p);
3915 }
3916
3917 static inline void
3918 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
3919 {
3920         struct thread *t = &p->threads[p->thread_id];
3921         struct instruction *ip = t->ip;
3922
3923         /* Structs. */
3924         __instr_regprefetch_rm_exec(p, t, ip);
3925
3926         /* Thread. */
3927         thread_ip_inc(p);
3928 }
3929
3930 static inline void
3931 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
3932 {
3933         struct thread *t = &p->threads[p->thread_id];
3934         struct instruction *ip = t->ip;
3935
3936         /* Structs. */
3937         __instr_regprefetch_ri_exec(p, t, ip);
3938
3939         /* Thread. */
3940         thread_ip_inc(p);
3941 }
3942
3943 static inline void
3944 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
3945 {
3946         struct thread *t = &p->threads[p->thread_id];
3947         struct instruction *ip = t->ip;
3948
3949         /* Structs. */
3950         __instr_regrd_hrh_exec(p, t, ip);
3951
3952         /* Thread. */
3953         thread_ip_inc(p);
3954 }
3955
3956 static inline void
3957 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
3958 {
3959         struct thread *t = &p->threads[p->thread_id];
3960         struct instruction *ip = t->ip;
3961
3962         /* Structs. */
3963         __instr_regrd_hrm_exec(p, t, ip);
3964
3965         /* Thread. */
3966         thread_ip_inc(p);
3967 }
3968
3969 static inline void
3970 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
3971 {
3972         struct thread *t = &p->threads[p->thread_id];
3973         struct instruction *ip = t->ip;
3974
3975         /* Structs. */
3976         __instr_regrd_mrh_exec(p, t, ip);
3977
3978         /* Thread. */
3979         thread_ip_inc(p);
3980 }
3981
3982 static inline void
3983 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
3984 {
3985         struct thread *t = &p->threads[p->thread_id];
3986         struct instruction *ip = t->ip;
3987
3988         /* Structs. */
3989         __instr_regrd_mrm_exec(p, t, ip);
3990
3991         /* Thread. */
3992         thread_ip_inc(p);
3993 }
3994
3995 static inline void
3996 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
3997 {
3998         struct thread *t = &p->threads[p->thread_id];
3999         struct instruction *ip = t->ip;
4000
4001         /* Structs. */
4002         __instr_regrd_hri_exec(p, t, ip);
4003
4004         /* Thread. */
4005         thread_ip_inc(p);
4006 }
4007
4008 static inline void
4009 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4010 {
4011         struct thread *t = &p->threads[p->thread_id];
4012         struct instruction *ip = t->ip;
4013
4014         /* Structs. */
4015         __instr_regrd_mri_exec(p, t, ip);
4016
4017         /* Thread. */
4018         thread_ip_inc(p);
4019 }
4020
4021 static inline void
4022 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4023 {
4024         struct thread *t = &p->threads[p->thread_id];
4025         struct instruction *ip = t->ip;
4026
4027         /* Structs. */
4028         __instr_regwr_rhh_exec(p, t, ip);
4029
4030         /* Thread. */
4031         thread_ip_inc(p);
4032 }
4033
4034 static inline void
4035 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4036 {
4037         struct thread *t = &p->threads[p->thread_id];
4038         struct instruction *ip = t->ip;
4039
4040         /* Structs. */
4041         __instr_regwr_rhm_exec(p, t, ip);
4042
4043         /* Thread. */
4044         thread_ip_inc(p);
4045 }
4046
4047 static inline void
4048 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4049 {
4050         struct thread *t = &p->threads[p->thread_id];
4051         struct instruction *ip = t->ip;
4052
4053         /* Structs. */
4054         __instr_regwr_rmh_exec(p, t, ip);
4055
4056         /* Thread. */
4057         thread_ip_inc(p);
4058 }
4059
4060 static inline void
4061 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4062 {
4063         struct thread *t = &p->threads[p->thread_id];
4064         struct instruction *ip = t->ip;
4065
4066         /* Structs. */
4067         __instr_regwr_rmm_exec(p, t, ip);
4068
4069         /* Thread. */
4070         thread_ip_inc(p);
4071 }
4072
4073 static inline void
4074 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4075 {
4076         struct thread *t = &p->threads[p->thread_id];
4077         struct instruction *ip = t->ip;
4078
4079         /* Structs. */
4080         __instr_regwr_rhi_exec(p, t, ip);
4081
4082         /* Thread. */
4083         thread_ip_inc(p);
4084 }
4085
4086 static inline void
4087 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4088 {
4089         struct thread *t = &p->threads[p->thread_id];
4090         struct instruction *ip = t->ip;
4091
4092         /* Structs. */
4093         __instr_regwr_rmi_exec(p, t, ip);
4094
4095         /* Thread. */
4096         thread_ip_inc(p);
4097 }
4098
4099 static inline void
4100 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4101 {
4102         struct thread *t = &p->threads[p->thread_id];
4103         struct instruction *ip = t->ip;
4104
4105         /* Structs. */
4106         __instr_regwr_rih_exec(p, t, ip);
4107
4108         /* Thread. */
4109         thread_ip_inc(p);
4110 }
4111
4112 static inline void
4113 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4114 {
4115         struct thread *t = &p->threads[p->thread_id];
4116         struct instruction *ip = t->ip;
4117
4118         /* Structs. */
4119         __instr_regwr_rim_exec(p, t, ip);
4120
4121         /* Thread. */
4122         thread_ip_inc(p);
4123 }
4124
4125 static inline void
4126 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4127 {
4128         struct thread *t = &p->threads[p->thread_id];
4129         struct instruction *ip = t->ip;
4130
4131         /* Structs. */
4132         __instr_regwr_rii_exec(p, t, ip);
4133
4134         /* Thread. */
4135         thread_ip_inc(p);
4136 }
4137
4138 static inline void
4139 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4140 {
4141         struct thread *t = &p->threads[p->thread_id];
4142         struct instruction *ip = t->ip;
4143
4144         /* Structs. */
4145         __instr_regadd_rhh_exec(p, t, ip);
4146
4147         /* Thread. */
4148         thread_ip_inc(p);
4149 }
4150
4151 static inline void
4152 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4153 {
4154         struct thread *t = &p->threads[p->thread_id];
4155         struct instruction *ip = t->ip;
4156
4157         /* Structs. */
4158         __instr_regadd_rhm_exec(p, t, ip);
4159
4160         /* Thread. */
4161         thread_ip_inc(p);
4162 }
4163
4164 static inline void
4165 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4166 {
4167         struct thread *t = &p->threads[p->thread_id];
4168         struct instruction *ip = t->ip;
4169
4170         /* Structs. */
4171         __instr_regadd_rmh_exec(p, t, ip);
4172
4173         /* Thread. */
4174         thread_ip_inc(p);
4175 }
4176
4177 static inline void
4178 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4179 {
4180         struct thread *t = &p->threads[p->thread_id];
4181         struct instruction *ip = t->ip;
4182
4183         /* Structs. */
4184         __instr_regadd_rmm_exec(p, t, ip);
4185
4186         /* Thread. */
4187         thread_ip_inc(p);
4188 }
4189
4190 static inline void
4191 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4192 {
4193         struct thread *t = &p->threads[p->thread_id];
4194         struct instruction *ip = t->ip;
4195
4196         /* Structs. */
4197         __instr_regadd_rhi_exec(p, t, ip);
4198
4199         /* Thread. */
4200         thread_ip_inc(p);
4201 }
4202
4203 static inline void
4204 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4205 {
4206         struct thread *t = &p->threads[p->thread_id];
4207         struct instruction *ip = t->ip;
4208
4209         /* Structs. */
4210         __instr_regadd_rmi_exec(p, t, ip);
4211
4212         /* Thread. */
4213         thread_ip_inc(p);
4214 }
4215
4216 static inline void
4217 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4218 {
4219         struct thread *t = &p->threads[p->thread_id];
4220         struct instruction *ip = t->ip;
4221
4222         /* Structs. */
4223         __instr_regadd_rih_exec(p, t, ip);
4224
4225         /* Thread. */
4226         thread_ip_inc(p);
4227 }
4228
4229 static inline void
4230 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4231 {
4232         struct thread *t = &p->threads[p->thread_id];
4233         struct instruction *ip = t->ip;
4234
4235         /* Structs. */
4236         __instr_regadd_rim_exec(p, t, ip);
4237
4238         /* Thread. */
4239         thread_ip_inc(p);
4240 }
4241
4242 static inline void
4243 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4244 {
4245         struct thread *t = &p->threads[p->thread_id];
4246         struct instruction *ip = t->ip;
4247
4248         /* Structs. */
4249         __instr_regadd_rii_exec(p, t, ip);
4250
4251         /* Thread. */
4252         thread_ip_inc(p);
4253 }
4254
4255 /*
4256  * metarray.
4257  */
4258 static struct metarray *
4259 metarray_find(struct rte_swx_pipeline *p, const char *name);
4260
4261 static int
4262 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4263                             struct action *action,
4264                             char **tokens,
4265                             int n_tokens,
4266                             struct instruction *instr,
4267                             struct instruction_data *data __rte_unused)
4268 {
4269         char *metarray = tokens[1], *idx = tokens[2];
4270         struct metarray *m;
4271         struct field *fidx;
4272         uint32_t idx_struct_id, idx_val;
4273
4274         CHECK(n_tokens == 3, EINVAL);
4275
4276         m = metarray_find(p, metarray);
4277         CHECK(m, EINVAL);
4278
4279         /* METPREFETCH_H, METPREFETCH_M. */
4280         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4281         if (fidx) {
4282                 CHECK(!fidx->var_size, EINVAL);
4283
4284                 instr->type = INSTR_METPREFETCH_M;
4285                 if (idx[0] == 'h')
4286                         instr->type = INSTR_METPREFETCH_H;
4287
4288                 instr->meter.metarray_id = m->id;
4289                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4290                 instr->meter.idx.n_bits = fidx->n_bits;
4291                 instr->meter.idx.offset = fidx->offset / 8;
4292                 return 0;
4293         }
4294
4295         /* METPREFETCH_I. */
4296         idx_val = strtoul(idx, &idx, 0);
4297         CHECK(!idx[0], EINVAL);
4298
4299         instr->type = INSTR_METPREFETCH_I;
4300         instr->meter.metarray_id = m->id;
4301         instr->meter.idx_val = idx_val;
4302         return 0;
4303 }
4304
4305 static int
4306 instr_meter_translate(struct rte_swx_pipeline *p,
4307                       struct action *action,
4308                       char **tokens,
4309                       int n_tokens,
4310                       struct instruction *instr,
4311                       struct instruction_data *data __rte_unused)
4312 {
4313         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4314         char *color_in = tokens[4], *color_out = tokens[5];
4315         struct metarray *m;
4316         struct field *fidx, *flength, *fcin, *fcout;
4317         uint32_t idx_struct_id, length_struct_id;
4318         uint32_t color_in_struct_id, color_out_struct_id;
4319
4320         CHECK(n_tokens == 6, EINVAL);
4321
4322         m = metarray_find(p, metarray);
4323         CHECK(m, EINVAL);
4324
4325         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4326
4327         flength = struct_field_parse(p, action, length, &length_struct_id);
4328         CHECK(flength, EINVAL);
4329         CHECK(!flength->var_size, EINVAL);
4330
4331         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4332
4333         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4334         CHECK(fcout, EINVAL);
4335         CHECK(!fcout->var_size, EINVAL);
4336
4337         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4338         if (fidx && fcin) {
4339                 CHECK(!fidx->var_size, EINVAL);
4340                 CHECK(!fcin->var_size, EINVAL);
4341
4342                 instr->type = INSTR_METER_MMM;
4343                 if (idx[0] == 'h' && length[0] == 'h')
4344                         instr->type = INSTR_METER_HHM;
4345                 if (idx[0] == 'h' && length[0] != 'h')
4346                         instr->type = INSTR_METER_HMM;
4347                 if (idx[0] != 'h' && length[0] == 'h')
4348                         instr->type = INSTR_METER_MHM;
4349
4350                 instr->meter.metarray_id = m->id;
4351
4352                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4353                 instr->meter.idx.n_bits = fidx->n_bits;
4354                 instr->meter.idx.offset = fidx->offset / 8;
4355
4356                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4357                 instr->meter.length.n_bits = flength->n_bits;
4358                 instr->meter.length.offset = flength->offset / 8;
4359
4360                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4361                 instr->meter.color_in.n_bits = fcin->n_bits;
4362                 instr->meter.color_in.offset = fcin->offset / 8;
4363
4364                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4365                 instr->meter.color_out.n_bits = fcout->n_bits;
4366                 instr->meter.color_out.offset = fcout->offset / 8;
4367
4368                 return 0;
4369         }
4370
4371         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4372         if (fidx && !fcin) {
4373                 uint32_t color_in_val;
4374
4375                 CHECK(!fidx->var_size, EINVAL);
4376
4377                 color_in_val = strtoul(color_in, &color_in, 0);
4378                 CHECK(!color_in[0], EINVAL);
4379
4380                 instr->type = INSTR_METER_MMI;
4381                 if (idx[0] == 'h' && length[0] == 'h')
4382                         instr->type = INSTR_METER_HHI;
4383                 if (idx[0] == 'h' && length[0] != 'h')
4384                         instr->type = INSTR_METER_HMI;
4385                 if (idx[0] != 'h' && length[0] == 'h')
4386                         instr->type = INSTR_METER_MHI;
4387
4388                 instr->meter.metarray_id = m->id;
4389
4390                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4391                 instr->meter.idx.n_bits = fidx->n_bits;
4392                 instr->meter.idx.offset = fidx->offset / 8;
4393
4394                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4395                 instr->meter.length.n_bits = flength->n_bits;
4396                 instr->meter.length.offset = flength->offset / 8;
4397
4398                 instr->meter.color_in_val = color_in_val;
4399
4400                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4401                 instr->meter.color_out.n_bits = fcout->n_bits;
4402                 instr->meter.color_out.offset = fcout->offset / 8;
4403
4404                 return 0;
4405         }
4406
4407         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4408         if (!fidx && fcin) {
4409                 uint32_t idx_val;
4410
4411                 idx_val = strtoul(idx, &idx, 0);
4412                 CHECK(!idx[0], EINVAL);
4413
4414                 CHECK(!fcin->var_size, EINVAL);
4415
4416                 instr->type = INSTR_METER_IMM;
4417                 if (length[0] == 'h')
4418                         instr->type = INSTR_METER_IHM;
4419
4420                 instr->meter.metarray_id = m->id;
4421
4422                 instr->meter.idx_val = idx_val;
4423
4424                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4425                 instr->meter.length.n_bits = flength->n_bits;
4426                 instr->meter.length.offset = flength->offset / 8;
4427
4428                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4429                 instr->meter.color_in.n_bits = fcin->n_bits;
4430                 instr->meter.color_in.offset = fcin->offset / 8;
4431
4432                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4433                 instr->meter.color_out.n_bits = fcout->n_bits;
4434                 instr->meter.color_out.offset = fcout->offset / 8;
4435
4436                 return 0;
4437         }
4438
4439         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4440         if (!fidx && !fcin) {
4441                 uint32_t idx_val, color_in_val;
4442
4443                 idx_val = strtoul(idx, &idx, 0);
4444                 CHECK(!idx[0], EINVAL);
4445
4446                 color_in_val = strtoul(color_in, &color_in, 0);
4447                 CHECK(!color_in[0], EINVAL);
4448
4449                 instr->type = INSTR_METER_IMI;
4450                 if (length[0] == 'h')
4451                         instr->type = INSTR_METER_IHI;
4452
4453                 instr->meter.metarray_id = m->id;
4454
4455                 instr->meter.idx_val = idx_val;
4456
4457                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4458                 instr->meter.length.n_bits = flength->n_bits;
4459                 instr->meter.length.offset = flength->offset / 8;
4460
4461                 instr->meter.color_in_val = color_in_val;
4462
4463                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4464                 instr->meter.color_out.n_bits = fcout->n_bits;
4465                 instr->meter.color_out.offset = fcout->offset / 8;
4466
4467                 return 0;
4468         }
4469
4470         CHECK(0, EINVAL);
4471 }
4472
4473 static inline struct meter *
4474 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4475 {
4476         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4477
4478         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4479         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4480         uint64_t idx64 = *idx64_ptr;
4481         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
4482         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4483
4484         return &r->metarray[idx];
4485 }
4486
4487 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4488
4489 static inline struct meter *
4490 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4491 {
4492         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4493
4494         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4495         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4496         uint64_t idx64 = *idx64_ptr;
4497         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
4498
4499         return &r->metarray[idx];
4500 }
4501
4502 #else
4503
4504 #define instr_meter_idx_nbo instr_meter_idx_hbo
4505
4506 #endif
4507
4508 static inline struct meter *
4509 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4510 {
4511         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4512
4513         uint64_t idx =  ip->meter.idx_val & r->size_mask;
4514
4515         return &r->metarray[idx];
4516 }
4517
4518 static inline uint32_t
4519 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
4520 {
4521         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4522         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4523         uint64_t src64 = *src64_ptr;
4524         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
4525         uint64_t src = src64 & src64_mask;
4526
4527         return (uint32_t)src;
4528 }
4529
4530 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4531
4532 static inline uint32_t
4533 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
4534 {
4535         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4536         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4537         uint64_t src64 = *src64_ptr;
4538         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
4539
4540         return (uint32_t)src;
4541 }
4542
4543 #else
4544
4545 #define instr_meter_length_nbo instr_meter_length_hbo
4546
4547 #endif
4548
4549 static inline enum rte_color
4550 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
4551 {
4552         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
4553         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
4554         uint64_t src64 = *src64_ptr;
4555         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
4556         uint64_t src = src64 & src64_mask;
4557
4558         return (enum rte_color)src;
4559 }
4560
4561 static inline void
4562 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
4563 {
4564         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
4565         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
4566         uint64_t dst64 = *dst64_ptr;
4567         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
4568
4569         uint64_t src = (uint64_t)color_out;
4570
4571         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4572 }
4573
4574 static inline void
4575 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4576 {
4577         struct thread *t = &p->threads[p->thread_id];
4578         struct instruction *ip = t->ip;
4579         struct meter *m;
4580
4581         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
4582
4583         /* Structs. */
4584         m = instr_meter_idx_nbo(p, t, ip);
4585         rte_prefetch0(m);
4586
4587         /* Thread. */
4588         thread_ip_inc(p);
4589 }
4590
4591 static inline void
4592 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4593 {
4594         struct thread *t = &p->threads[p->thread_id];
4595         struct instruction *ip = t->ip;
4596         struct meter *m;
4597
4598         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
4599
4600         /* Structs. */
4601         m = instr_meter_idx_hbo(p, t, ip);
4602         rte_prefetch0(m);
4603
4604         /* Thread. */
4605         thread_ip_inc(p);
4606 }
4607
4608 static inline void
4609 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4610 {
4611         struct thread *t = &p->threads[p->thread_id];
4612         struct instruction *ip = t->ip;
4613         struct meter *m;
4614
4615         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
4616
4617         /* Structs. */
4618         m = instr_meter_idx_imm(p, ip);
4619         rte_prefetch0(m);
4620
4621         /* Thread. */
4622         thread_ip_inc(p);
4623 }
4624
4625 static inline void
4626 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4627 {
4628         struct thread *t = &p->threads[p->thread_id];
4629         struct instruction *ip = t->ip;
4630         struct meter *m;
4631         uint64_t time, n_pkts, n_bytes;
4632         uint32_t length;
4633         enum rte_color color_in, color_out;
4634
4635         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
4636
4637         /* Structs. */
4638         m = instr_meter_idx_nbo(p, t, ip);
4639         rte_prefetch0(m->n_pkts);
4640         time = rte_get_tsc_cycles();
4641         length = instr_meter_length_nbo(t, ip);
4642         color_in = instr_meter_color_in_hbo(t, ip);
4643
4644         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4645                 &m->profile->profile,
4646                 time,
4647                 length,
4648                 color_in);
4649
4650         color_out &= m->color_mask;
4651
4652         n_pkts = m->n_pkts[color_out];
4653         n_bytes = m->n_bytes[color_out];
4654
4655         instr_meter_color_out_hbo_set(t, ip, color_out);
4656
4657         m->n_pkts[color_out] = n_pkts + 1;
4658         m->n_bytes[color_out] = n_bytes + length;
4659
4660         /* Thread. */
4661         thread_ip_inc(p);
4662 }
4663
4664 static inline void
4665 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4666 {
4667         struct thread *t = &p->threads[p->thread_id];
4668         struct instruction *ip = t->ip;
4669         struct meter *m;
4670         uint64_t time, n_pkts, n_bytes;
4671         uint32_t length;
4672         enum rte_color color_in, color_out;
4673
4674         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
4675
4676         /* Structs. */
4677         m = instr_meter_idx_nbo(p, t, ip);
4678         rte_prefetch0(m->n_pkts);
4679         time = rte_get_tsc_cycles();
4680         length = instr_meter_length_nbo(t, ip);
4681         color_in = (enum rte_color)ip->meter.color_in_val;
4682
4683         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4684                 &m->profile->profile,
4685                 time,
4686                 length,
4687                 color_in);
4688
4689         color_out &= m->color_mask;
4690
4691         n_pkts = m->n_pkts[color_out];
4692         n_bytes = m->n_bytes[color_out];
4693
4694         instr_meter_color_out_hbo_set(t, ip, color_out);
4695
4696         m->n_pkts[color_out] = n_pkts + 1;
4697         m->n_bytes[color_out] = n_bytes + length;
4698
4699         /* Thread. */
4700         thread_ip_inc(p);
4701 }
4702
4703 static inline void
4704 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4705 {
4706         struct thread *t = &p->threads[p->thread_id];
4707         struct instruction *ip = t->ip;
4708         struct meter *m;
4709         uint64_t time, n_pkts, n_bytes;
4710         uint32_t length;
4711         enum rte_color color_in, color_out;
4712
4713         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
4714
4715         /* Structs. */
4716         m = instr_meter_idx_nbo(p, t, ip);
4717         rte_prefetch0(m->n_pkts);
4718         time = rte_get_tsc_cycles();
4719         length = instr_meter_length_hbo(t, ip);
4720         color_in = instr_meter_color_in_hbo(t, ip);
4721
4722         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4723                 &m->profile->profile,
4724                 time,
4725                 length,
4726                 color_in);
4727
4728         color_out &= m->color_mask;
4729
4730         n_pkts = m->n_pkts[color_out];
4731         n_bytes = m->n_bytes[color_out];
4732
4733         instr_meter_color_out_hbo_set(t, ip, color_out);
4734
4735         m->n_pkts[color_out] = n_pkts + 1;
4736         m->n_bytes[color_out] = n_bytes + length;
4737
4738         /* Thread. */
4739         thread_ip_inc(p);
4740 }
4741 static inline void
4742 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4743 {
4744         struct thread *t = &p->threads[p->thread_id];
4745         struct instruction *ip = t->ip;
4746         struct meter *m;
4747         uint64_t time, n_pkts, n_bytes;
4748         uint32_t length;
4749         enum rte_color color_in, color_out;
4750
4751         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
4752
4753         /* Structs. */
4754         m = instr_meter_idx_nbo(p, t, ip);
4755         rte_prefetch0(m->n_pkts);
4756         time = rte_get_tsc_cycles();
4757         length = instr_meter_length_hbo(t, ip);
4758         color_in = (enum rte_color)ip->meter.color_in_val;
4759
4760         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4761                 &m->profile->profile,
4762                 time,
4763                 length,
4764                 color_in);
4765
4766         color_out &= m->color_mask;
4767
4768         n_pkts = m->n_pkts[color_out];
4769         n_bytes = m->n_bytes[color_out];
4770
4771         instr_meter_color_out_hbo_set(t, ip, color_out);
4772
4773         m->n_pkts[color_out] = n_pkts + 1;
4774         m->n_bytes[color_out] = n_bytes + length;
4775
4776         /* Thread. */
4777         thread_ip_inc(p);
4778 }
4779
4780 static inline void
4781 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4782 {
4783         struct thread *t = &p->threads[p->thread_id];
4784         struct instruction *ip = t->ip;
4785         struct meter *m;
4786         uint64_t time, n_pkts, n_bytes;
4787         uint32_t length;
4788         enum rte_color color_in, color_out;
4789
4790         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
4791
4792         /* Structs. */
4793         m = instr_meter_idx_hbo(p, t, ip);
4794         rte_prefetch0(m->n_pkts);
4795         time = rte_get_tsc_cycles();
4796         length = instr_meter_length_nbo(t, ip);
4797         color_in = instr_meter_color_in_hbo(t, ip);
4798
4799         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4800                 &m->profile->profile,
4801                 time,
4802                 length,
4803                 color_in);
4804
4805         color_out &= m->color_mask;
4806
4807         n_pkts = m->n_pkts[color_out];
4808         n_bytes = m->n_bytes[color_out];
4809
4810         instr_meter_color_out_hbo_set(t, ip, color_out);
4811
4812         m->n_pkts[color_out] = n_pkts + 1;
4813         m->n_bytes[color_out] = n_bytes + length;
4814
4815         /* Thread. */
4816         thread_ip_inc(p);
4817 }
4818
4819 static inline void
4820 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4821 {
4822         struct thread *t = &p->threads[p->thread_id];
4823         struct instruction *ip = t->ip;
4824         struct meter *m;
4825         uint64_t time, n_pkts, n_bytes;
4826         uint32_t length;
4827         enum rte_color color_in, color_out;
4828
4829         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4830
4831         /* Structs. */
4832         m = instr_meter_idx_hbo(p, t, ip);
4833         rte_prefetch0(m->n_pkts);
4834         time = rte_get_tsc_cycles();
4835         length = instr_meter_length_nbo(t, ip);
4836         color_in = (enum rte_color)ip->meter.color_in_val;
4837
4838         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4839                 &m->profile->profile,
4840                 time,
4841                 length,
4842                 color_in);
4843
4844         color_out &= m->color_mask;
4845
4846         n_pkts = m->n_pkts[color_out];
4847         n_bytes = m->n_bytes[color_out];
4848
4849         instr_meter_color_out_hbo_set(t, ip, color_out);
4850
4851         m->n_pkts[color_out] = n_pkts + 1;
4852         m->n_bytes[color_out] = n_bytes + length;
4853
4854         /* Thread. */
4855         thread_ip_inc(p);
4856 }
4857
4858 static inline void
4859 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4860 {
4861         struct thread *t = &p->threads[p->thread_id];
4862         struct instruction *ip = t->ip;
4863         struct meter *m;
4864         uint64_t time, n_pkts, n_bytes;
4865         uint32_t length;
4866         enum rte_color color_in, color_out;
4867
4868         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4869
4870         /* Structs. */
4871         m = instr_meter_idx_hbo(p, t, ip);
4872         rte_prefetch0(m->n_pkts);
4873         time = rte_get_tsc_cycles();
4874         length = instr_meter_length_hbo(t, ip);
4875         color_in = instr_meter_color_in_hbo(t, ip);
4876
4877         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4878                 &m->profile->profile,
4879                 time,
4880                 length,
4881                 color_in);
4882
4883         color_out &= m->color_mask;
4884
4885         n_pkts = m->n_pkts[color_out];
4886         n_bytes = m->n_bytes[color_out];
4887
4888         instr_meter_color_out_hbo_set(t, ip, color_out);
4889
4890         m->n_pkts[color_out] = n_pkts + 1;
4891         m->n_bytes[color_out] = n_bytes + length;
4892
4893         /* Thread. */
4894         thread_ip_inc(p);
4895 }
4896
4897 static inline void
4898 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4899 {
4900         struct thread *t = &p->threads[p->thread_id];
4901         struct instruction *ip = t->ip;
4902         struct meter *m;
4903         uint64_t time, n_pkts, n_bytes;
4904         uint32_t length;
4905         enum rte_color color_in, color_out;
4906
4907         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4908
4909         /* Structs. */
4910         m = instr_meter_idx_hbo(p, t, ip);
4911         rte_prefetch0(m->n_pkts);
4912         time = rte_get_tsc_cycles();
4913         length = instr_meter_length_hbo(t, ip);
4914         color_in = (enum rte_color)ip->meter.color_in_val;
4915
4916         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4917                 &m->profile->profile,
4918                 time,
4919                 length,
4920                 color_in);
4921
4922         color_out &= m->color_mask;
4923
4924         n_pkts = m->n_pkts[color_out];
4925         n_bytes = m->n_bytes[color_out];
4926
4927         instr_meter_color_out_hbo_set(t, ip, color_out);
4928
4929         m->n_pkts[color_out] = n_pkts + 1;
4930         m->n_bytes[color_out] = n_bytes + length;
4931
4932         /* Thread. */
4933         thread_ip_inc(p);
4934 }
4935
4936 static inline void
4937 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4938 {
4939         struct thread *t = &p->threads[p->thread_id];
4940         struct instruction *ip = t->ip;
4941         struct meter *m;
4942         uint64_t time, n_pkts, n_bytes;
4943         uint32_t length;
4944         enum rte_color color_in, color_out;
4945
4946         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4947
4948         /* Structs. */
4949         m = instr_meter_idx_imm(p, ip);
4950         rte_prefetch0(m->n_pkts);
4951         time = rte_get_tsc_cycles();
4952         length = instr_meter_length_nbo(t, ip);
4953         color_in = instr_meter_color_in_hbo(t, ip);
4954
4955         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4956                 &m->profile->profile,
4957                 time,
4958                 length,
4959                 color_in);
4960
4961         color_out &= m->color_mask;
4962
4963         n_pkts = m->n_pkts[color_out];
4964         n_bytes = m->n_bytes[color_out];
4965
4966         instr_meter_color_out_hbo_set(t, ip, color_out);
4967
4968         m->n_pkts[color_out] = n_pkts + 1;
4969         m->n_bytes[color_out] = n_bytes + length;
4970
4971         /* Thread. */
4972         thread_ip_inc(p);
4973 }
4974
4975 static inline void
4976 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4977 {
4978         struct thread *t = &p->threads[p->thread_id];
4979         struct instruction *ip = t->ip;
4980         struct meter *m;
4981         uint64_t time, n_pkts, n_bytes;
4982         uint32_t length;
4983         enum rte_color color_in, color_out;
4984
4985         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4986
4987         /* Structs. */
4988         m = instr_meter_idx_imm(p, ip);
4989         rte_prefetch0(m->n_pkts);
4990         time = rte_get_tsc_cycles();
4991         length = instr_meter_length_nbo(t, ip);
4992         color_in = (enum rte_color)ip->meter.color_in_val;
4993
4994         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4995                 &m->profile->profile,
4996                 time,
4997                 length,
4998                 color_in);
4999
5000         color_out &= m->color_mask;
5001
5002         n_pkts = m->n_pkts[color_out];
5003         n_bytes = m->n_bytes[color_out];
5004
5005         instr_meter_color_out_hbo_set(t, ip, color_out);
5006
5007         m->n_pkts[color_out] = n_pkts + 1;
5008         m->n_bytes[color_out] = n_bytes + length;
5009
5010         /* Thread. */
5011         thread_ip_inc(p);
5012 }
5013
5014 static inline void
5015 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5016 {
5017         struct thread *t = &p->threads[p->thread_id];
5018         struct instruction *ip = t->ip;
5019         struct meter *m;
5020         uint64_t time, n_pkts, n_bytes;
5021         uint32_t length;
5022         enum rte_color color_in, color_out;
5023
5024         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5025
5026         /* Structs. */
5027         m = instr_meter_idx_imm(p, ip);
5028         rte_prefetch0(m->n_pkts);
5029         time = rte_get_tsc_cycles();
5030         length = instr_meter_length_hbo(t, ip);
5031         color_in = instr_meter_color_in_hbo(t, ip);
5032
5033         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5034                 &m->profile->profile,
5035                 time,
5036                 length,
5037                 color_in);
5038
5039         color_out &= m->color_mask;
5040
5041         n_pkts = m->n_pkts[color_out];
5042         n_bytes = m->n_bytes[color_out];
5043
5044         instr_meter_color_out_hbo_set(t, ip, color_out);
5045
5046         m->n_pkts[color_out] = n_pkts + 1;
5047         m->n_bytes[color_out] = n_bytes + length;
5048
5049         /* Thread. */
5050         thread_ip_inc(p);
5051 }
5052 static inline void
5053 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5054 {
5055         struct thread *t = &p->threads[p->thread_id];
5056         struct instruction *ip = t->ip;
5057         struct meter *m;
5058         uint64_t time, n_pkts, n_bytes;
5059         uint32_t length;
5060         enum rte_color color_in, color_out;
5061
5062         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5063
5064         /* Structs. */
5065         m = instr_meter_idx_imm(p, ip);
5066         rte_prefetch0(m->n_pkts);
5067         time = rte_get_tsc_cycles();
5068         length = instr_meter_length_hbo(t, ip);
5069         color_in = (enum rte_color)ip->meter.color_in_val;
5070
5071         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5072                 &m->profile->profile,
5073                 time,
5074                 length,
5075                 color_in);
5076
5077         color_out &= m->color_mask;
5078
5079         n_pkts = m->n_pkts[color_out];
5080         n_bytes = m->n_bytes[color_out];
5081
5082         instr_meter_color_out_hbo_set(t, ip, color_out);
5083
5084         m->n_pkts[color_out] = n_pkts + 1;
5085         m->n_bytes[color_out] = n_bytes + length;
5086
5087         /* Thread. */
5088         thread_ip_inc(p);
5089 }
5090
5091 /*
5092  * jmp.
5093  */
5094 static int
5095 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5096                     struct action *action __rte_unused,
5097                     char **tokens,
5098                     int n_tokens,
5099                     struct instruction *instr,
5100                     struct instruction_data *data)
5101 {
5102         CHECK(n_tokens == 2, EINVAL);
5103
5104         strcpy(data->jmp_label, tokens[1]);
5105
5106         instr->type = INSTR_JMP;
5107         instr->jmp.ip = NULL; /* Resolved later. */
5108         return 0;
5109 }
5110
5111 static int
5112 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5113                           struct action *action __rte_unused,
5114                           char **tokens,
5115                           int n_tokens,
5116                           struct instruction *instr,
5117                           struct instruction_data *data)
5118 {
5119         struct header *h;
5120
5121         CHECK(n_tokens == 3, EINVAL);
5122
5123         strcpy(data->jmp_label, tokens[1]);
5124
5125         h = header_parse(p, tokens[2]);
5126         CHECK(h, EINVAL);
5127
5128         instr->type = INSTR_JMP_VALID;
5129         instr->jmp.ip = NULL; /* Resolved later. */
5130         instr->jmp.header_id = h->id;
5131         return 0;
5132 }
5133
5134 static int
5135 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5136                             struct action *action __rte_unused,
5137                             char **tokens,
5138                             int n_tokens,
5139                             struct instruction *instr,
5140                             struct instruction_data *data)
5141 {
5142         struct header *h;
5143
5144         CHECK(n_tokens == 3, EINVAL);
5145
5146         strcpy(data->jmp_label, tokens[1]);
5147
5148         h = header_parse(p, tokens[2]);
5149         CHECK(h, EINVAL);
5150
5151         instr->type = INSTR_JMP_INVALID;
5152         instr->jmp.ip = NULL; /* Resolved later. */
5153         instr->jmp.header_id = h->id;
5154         return 0;
5155 }
5156
5157 static int
5158 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5159                         struct action *action,
5160                         char **tokens,
5161                         int n_tokens,
5162                         struct instruction *instr,
5163                         struct instruction_data *data)
5164 {
5165         CHECK(!action, EINVAL);
5166         CHECK(n_tokens == 2, EINVAL);
5167
5168         strcpy(data->jmp_label, tokens[1]);
5169
5170         instr->type = INSTR_JMP_HIT;
5171         instr->jmp.ip = NULL; /* Resolved later. */
5172         return 0;
5173 }
5174
5175 static int
5176 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5177                          struct action *action,
5178                          char **tokens,
5179                          int n_tokens,
5180                          struct instruction *instr,
5181                          struct instruction_data *data)
5182 {
5183         CHECK(!action, EINVAL);
5184         CHECK(n_tokens == 2, EINVAL);
5185
5186         strcpy(data->jmp_label, tokens[1]);
5187
5188         instr->type = INSTR_JMP_MISS;
5189         instr->jmp.ip = NULL; /* Resolved later. */
5190         return 0;
5191 }
5192
5193 static int
5194 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5195                                struct action *action,
5196                                char **tokens,
5197                                int n_tokens,
5198                                struct instruction *instr,
5199                                struct instruction_data *data)
5200 {
5201         struct action *a;
5202
5203         CHECK(!action, EINVAL);
5204         CHECK(n_tokens == 3, EINVAL);
5205
5206         strcpy(data->jmp_label, tokens[1]);
5207
5208         a = action_find(p, tokens[2]);
5209         CHECK(a, EINVAL);
5210
5211         instr->type = INSTR_JMP_ACTION_HIT;
5212         instr->jmp.ip = NULL; /* Resolved later. */
5213         instr->jmp.action_id = a->id;
5214         return 0;
5215 }
5216
5217 static int
5218 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5219                                 struct action *action,
5220                                 char **tokens,
5221                                 int n_tokens,
5222                                 struct instruction *instr,
5223                                 struct instruction_data *data)
5224 {
5225         struct action *a;
5226
5227         CHECK(!action, EINVAL);
5228         CHECK(n_tokens == 3, EINVAL);
5229
5230         strcpy(data->jmp_label, tokens[1]);
5231
5232         a = action_find(p, tokens[2]);
5233         CHECK(a, EINVAL);
5234
5235         instr->type = INSTR_JMP_ACTION_MISS;
5236         instr->jmp.ip = NULL; /* Resolved later. */
5237         instr->jmp.action_id = a->id;
5238         return 0;
5239 }
5240
5241 static int
5242 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5243                        struct action *action,
5244                        char **tokens,
5245                        int n_tokens,
5246                        struct instruction *instr,
5247                        struct instruction_data *data)
5248 {
5249         char *a = tokens[2], *b = tokens[3];
5250         struct field *fa, *fb;
5251         uint64_t b_val;
5252         uint32_t a_struct_id, b_struct_id;
5253
5254         CHECK(n_tokens == 4, EINVAL);
5255
5256         strcpy(data->jmp_label, tokens[1]);
5257
5258         fa = struct_field_parse(p, action, a, &a_struct_id);
5259         CHECK(fa, EINVAL);
5260         CHECK(!fa->var_size, EINVAL);
5261
5262         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5263         fb = struct_field_parse(p, action, b, &b_struct_id);
5264         if (fb) {
5265                 CHECK(!fb->var_size, EINVAL);
5266
5267                 instr->type = INSTR_JMP_EQ;
5268                 if (a[0] != 'h' && b[0] == 'h')
5269                         instr->type = INSTR_JMP_EQ_MH;
5270                 if (a[0] == 'h' && b[0] != 'h')
5271                         instr->type = INSTR_JMP_EQ_HM;
5272                 if (a[0] == 'h' && b[0] == 'h')
5273                         instr->type = INSTR_JMP_EQ_HH;
5274                 instr->jmp.ip = NULL; /* Resolved later. */
5275
5276                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5277                 instr->jmp.a.n_bits = fa->n_bits;
5278                 instr->jmp.a.offset = fa->offset / 8;
5279                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5280                 instr->jmp.b.n_bits = fb->n_bits;
5281                 instr->jmp.b.offset = fb->offset / 8;
5282                 return 0;
5283         }
5284
5285         /* JMP_EQ_I. */
5286         b_val = strtoull(b, &b, 0);
5287         CHECK(!b[0], EINVAL);
5288
5289         if (a[0] == 'h')
5290                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5291
5292         instr->type = INSTR_JMP_EQ_I;
5293         instr->jmp.ip = NULL; /* Resolved later. */
5294         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5295         instr->jmp.a.n_bits = fa->n_bits;
5296         instr->jmp.a.offset = fa->offset / 8;
5297         instr->jmp.b_val = b_val;
5298         return 0;
5299 }
5300
5301 static int
5302 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5303                         struct action *action,
5304                         char **tokens,
5305                         int n_tokens,
5306                         struct instruction *instr,
5307                         struct instruction_data *data)
5308 {
5309         char *a = tokens[2], *b = tokens[3];
5310         struct field *fa, *fb;
5311         uint64_t b_val;
5312         uint32_t a_struct_id, b_struct_id;
5313
5314         CHECK(n_tokens == 4, EINVAL);
5315
5316         strcpy(data->jmp_label, tokens[1]);
5317
5318         fa = struct_field_parse(p, action, a, &a_struct_id);
5319         CHECK(fa, EINVAL);
5320         CHECK(!fa->var_size, EINVAL);
5321
5322         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5323         fb = struct_field_parse(p, action, b, &b_struct_id);
5324         if (fb) {
5325                 CHECK(!fb->var_size, EINVAL);
5326
5327                 instr->type = INSTR_JMP_NEQ;
5328                 if (a[0] != 'h' && b[0] == 'h')
5329                         instr->type = INSTR_JMP_NEQ_MH;
5330                 if (a[0] == 'h' && b[0] != 'h')
5331                         instr->type = INSTR_JMP_NEQ_HM;
5332                 if (a[0] == 'h' && b[0] == 'h')
5333                         instr->type = INSTR_JMP_NEQ_HH;
5334                 instr->jmp.ip = NULL; /* Resolved later. */
5335
5336                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5337                 instr->jmp.a.n_bits = fa->n_bits;
5338                 instr->jmp.a.offset = fa->offset / 8;
5339                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5340                 instr->jmp.b.n_bits = fb->n_bits;
5341                 instr->jmp.b.offset = fb->offset / 8;
5342                 return 0;
5343         }
5344
5345         /* JMP_NEQ_I. */
5346         b_val = strtoull(b, &b, 0);
5347         CHECK(!b[0], EINVAL);
5348
5349         if (a[0] == 'h')
5350                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5351
5352         instr->type = INSTR_JMP_NEQ_I;
5353         instr->jmp.ip = NULL; /* Resolved later. */
5354         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5355         instr->jmp.a.n_bits = fa->n_bits;
5356         instr->jmp.a.offset = fa->offset / 8;
5357         instr->jmp.b_val = b_val;
5358         return 0;
5359 }
5360
5361 static int
5362 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5363                        struct action *action,
5364                        char **tokens,
5365                        int n_tokens,
5366                        struct instruction *instr,
5367                        struct instruction_data *data)
5368 {
5369         char *a = tokens[2], *b = tokens[3];
5370         struct field *fa, *fb;
5371         uint64_t b_val;
5372         uint32_t a_struct_id, b_struct_id;
5373
5374         CHECK(n_tokens == 4, EINVAL);
5375
5376         strcpy(data->jmp_label, tokens[1]);
5377
5378         fa = struct_field_parse(p, action, a, &a_struct_id);
5379         CHECK(fa, EINVAL);
5380         CHECK(!fa->var_size, EINVAL);
5381
5382         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5383         fb = struct_field_parse(p, action, b, &b_struct_id);
5384         if (fb) {
5385                 CHECK(!fb->var_size, EINVAL);
5386
5387                 instr->type = INSTR_JMP_LT;
5388                 if (a[0] == 'h' && b[0] != 'h')
5389                         instr->type = INSTR_JMP_LT_HM;
5390                 if (a[0] != 'h' && b[0] == 'h')
5391                         instr->type = INSTR_JMP_LT_MH;
5392                 if (a[0] == 'h' && b[0] == 'h')
5393                         instr->type = INSTR_JMP_LT_HH;
5394                 instr->jmp.ip = NULL; /* Resolved later. */
5395
5396                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5397                 instr->jmp.a.n_bits = fa->n_bits;
5398                 instr->jmp.a.offset = fa->offset / 8;
5399                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5400                 instr->jmp.b.n_bits = fb->n_bits;
5401                 instr->jmp.b.offset = fb->offset / 8;
5402                 return 0;
5403         }
5404
5405         /* JMP_LT_MI, JMP_LT_HI. */
5406         b_val = strtoull(b, &b, 0);
5407         CHECK(!b[0], EINVAL);
5408
5409         instr->type = INSTR_JMP_LT_MI;
5410         if (a[0] == 'h')
5411                 instr->type = INSTR_JMP_LT_HI;
5412         instr->jmp.ip = NULL; /* Resolved later. */
5413
5414         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5415         instr->jmp.a.n_bits = fa->n_bits;
5416         instr->jmp.a.offset = fa->offset / 8;
5417         instr->jmp.b_val = b_val;
5418         return 0;
5419 }
5420
5421 static int
5422 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5423                        struct action *action,
5424                        char **tokens,
5425                        int n_tokens,
5426                        struct instruction *instr,
5427                        struct instruction_data *data)
5428 {
5429         char *a = tokens[2], *b = tokens[3];
5430         struct field *fa, *fb;
5431         uint64_t b_val;
5432         uint32_t a_struct_id, b_struct_id;
5433
5434         CHECK(n_tokens == 4, EINVAL);
5435
5436         strcpy(data->jmp_label, tokens[1]);
5437
5438         fa = struct_field_parse(p, action, a, &a_struct_id);
5439         CHECK(fa, EINVAL);
5440         CHECK(!fa->var_size, EINVAL);
5441
5442         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5443         fb = struct_field_parse(p, action, b, &b_struct_id);
5444         if (fb) {
5445                 CHECK(!fb->var_size, EINVAL);
5446
5447                 instr->type = INSTR_JMP_GT;
5448                 if (a[0] == 'h' && b[0] != 'h')
5449                         instr->type = INSTR_JMP_GT_HM;
5450                 if (a[0] != 'h' && b[0] == 'h')
5451                         instr->type = INSTR_JMP_GT_MH;
5452                 if (a[0] == 'h' && b[0] == 'h')
5453                         instr->type = INSTR_JMP_GT_HH;
5454                 instr->jmp.ip = NULL; /* Resolved later. */
5455
5456                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5457                 instr->jmp.a.n_bits = fa->n_bits;
5458                 instr->jmp.a.offset = fa->offset / 8;
5459                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5460                 instr->jmp.b.n_bits = fb->n_bits;
5461                 instr->jmp.b.offset = fb->offset / 8;
5462                 return 0;
5463         }
5464
5465         /* JMP_GT_MI, JMP_GT_HI. */
5466         b_val = strtoull(b, &b, 0);
5467         CHECK(!b[0], EINVAL);
5468
5469         instr->type = INSTR_JMP_GT_MI;
5470         if (a[0] == 'h')
5471                 instr->type = INSTR_JMP_GT_HI;
5472         instr->jmp.ip = NULL; /* Resolved later. */
5473
5474         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5475         instr->jmp.a.n_bits = fa->n_bits;
5476         instr->jmp.a.offset = fa->offset / 8;
5477         instr->jmp.b_val = b_val;
5478         return 0;
5479 }
5480
5481 static inline void
5482 instr_jmp_exec(struct rte_swx_pipeline *p)
5483 {
5484         struct thread *t = &p->threads[p->thread_id];
5485         struct instruction *ip = t->ip;
5486
5487         TRACE("[Thread %2u] jmp\n", p->thread_id);
5488
5489         thread_ip_set(t, ip->jmp.ip);
5490 }
5491
5492 static inline void
5493 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5494 {
5495         struct thread *t = &p->threads[p->thread_id];
5496         struct instruction *ip = t->ip;
5497         uint32_t header_id = ip->jmp.header_id;
5498
5499         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5500
5501         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5502 }
5503
5504 static inline void
5505 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5506 {
5507         struct thread *t = &p->threads[p->thread_id];
5508         struct instruction *ip = t->ip;
5509         uint32_t header_id = ip->jmp.header_id;
5510
5511         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5512
5513         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5514 }
5515
5516 static inline void
5517 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5518 {
5519         struct thread *t = &p->threads[p->thread_id];
5520         struct instruction *ip = t->ip;
5521         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5522
5523         TRACE("[Thread %2u] jmph\n", p->thread_id);
5524
5525         t->ip = ip_next[t->hit];
5526 }
5527
5528 static inline void
5529 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5530 {
5531         struct thread *t = &p->threads[p->thread_id];
5532         struct instruction *ip = t->ip;
5533         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5534
5535         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5536
5537         t->ip = ip_next[t->hit];
5538 }
5539
5540 static inline void
5541 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5542 {
5543         struct thread *t = &p->threads[p->thread_id];
5544         struct instruction *ip = t->ip;
5545
5546         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5547
5548         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5549 }
5550
5551 static inline void
5552 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5553 {
5554         struct thread *t = &p->threads[p->thread_id];
5555         struct instruction *ip = t->ip;
5556
5557         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5558
5559         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5560 }
5561
5562 static inline void
5563 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5564 {
5565         struct thread *t = &p->threads[p->thread_id];
5566         struct instruction *ip = t->ip;
5567
5568         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5569
5570         JMP_CMP(t, ip, ==);
5571 }
5572
5573 static inline void
5574 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5575 {
5576         struct thread *t = &p->threads[p->thread_id];
5577         struct instruction *ip = t->ip;
5578
5579         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5580
5581         JMP_CMP_MH(t, ip, ==);
5582 }
5583
5584 static inline void
5585 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5586 {
5587         struct thread *t = &p->threads[p->thread_id];
5588         struct instruction *ip = t->ip;
5589
5590         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5591
5592         JMP_CMP_HM(t, ip, ==);
5593 }
5594
5595 static inline void
5596 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5597 {
5598         struct thread *t = &p->threads[p->thread_id];
5599         struct instruction *ip = t->ip;
5600
5601         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5602
5603         JMP_CMP_HH_FAST(t, ip, ==);
5604 }
5605
5606 static inline void
5607 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5608 {
5609         struct thread *t = &p->threads[p->thread_id];
5610         struct instruction *ip = t->ip;
5611
5612         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5613
5614         JMP_CMP_I(t, ip, ==);
5615 }
5616
5617 static inline void
5618 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5619 {
5620         struct thread *t = &p->threads[p->thread_id];
5621         struct instruction *ip = t->ip;
5622
5623         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5624
5625         JMP_CMP(t, ip, !=);
5626 }
5627
5628 static inline void
5629 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5630 {
5631         struct thread *t = &p->threads[p->thread_id];
5632         struct instruction *ip = t->ip;
5633
5634         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5635
5636         JMP_CMP_MH(t, ip, !=);
5637 }
5638
5639 static inline void
5640 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5641 {
5642         struct thread *t = &p->threads[p->thread_id];
5643         struct instruction *ip = t->ip;
5644
5645         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5646
5647         JMP_CMP_HM(t, ip, !=);
5648 }
5649
5650 static inline void
5651 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5652 {
5653         struct thread *t = &p->threads[p->thread_id];
5654         struct instruction *ip = t->ip;
5655
5656         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5657
5658         JMP_CMP_HH_FAST(t, ip, !=);
5659 }
5660
5661 static inline void
5662 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5663 {
5664         struct thread *t = &p->threads[p->thread_id];
5665         struct instruction *ip = t->ip;
5666
5667         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5668
5669         JMP_CMP_I(t, ip, !=);
5670 }
5671
5672 static inline void
5673 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5674 {
5675         struct thread *t = &p->threads[p->thread_id];
5676         struct instruction *ip = t->ip;
5677
5678         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5679
5680         JMP_CMP(t, ip, <);
5681 }
5682
5683 static inline void
5684 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5685 {
5686         struct thread *t = &p->threads[p->thread_id];
5687         struct instruction *ip = t->ip;
5688
5689         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5690
5691         JMP_CMP_MH(t, ip, <);
5692 }
5693
5694 static inline void
5695 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5696 {
5697         struct thread *t = &p->threads[p->thread_id];
5698         struct instruction *ip = t->ip;
5699
5700         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5701
5702         JMP_CMP_HM(t, ip, <);
5703 }
5704
5705 static inline void
5706 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5707 {
5708         struct thread *t = &p->threads[p->thread_id];
5709         struct instruction *ip = t->ip;
5710
5711         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5712
5713         JMP_CMP_HH(t, ip, <);
5714 }
5715
5716 static inline void
5717 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5718 {
5719         struct thread *t = &p->threads[p->thread_id];
5720         struct instruction *ip = t->ip;
5721
5722         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5723
5724         JMP_CMP_MI(t, ip, <);
5725 }
5726
5727 static inline void
5728 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5729 {
5730         struct thread *t = &p->threads[p->thread_id];
5731         struct instruction *ip = t->ip;
5732
5733         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5734
5735         JMP_CMP_HI(t, ip, <);
5736 }
5737
5738 static inline void
5739 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5740 {
5741         struct thread *t = &p->threads[p->thread_id];
5742         struct instruction *ip = t->ip;
5743
5744         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5745
5746         JMP_CMP(t, ip, >);
5747 }
5748
5749 static inline void
5750 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5751 {
5752         struct thread *t = &p->threads[p->thread_id];
5753         struct instruction *ip = t->ip;
5754
5755         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5756
5757         JMP_CMP_MH(t, ip, >);
5758 }
5759
5760 static inline void
5761 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5762 {
5763         struct thread *t = &p->threads[p->thread_id];
5764         struct instruction *ip = t->ip;
5765
5766         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5767
5768         JMP_CMP_HM(t, ip, >);
5769 }
5770
5771 static inline void
5772 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5773 {
5774         struct thread *t = &p->threads[p->thread_id];
5775         struct instruction *ip = t->ip;
5776
5777         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5778
5779         JMP_CMP_HH(t, ip, >);
5780 }
5781
5782 static inline void
5783 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5784 {
5785         struct thread *t = &p->threads[p->thread_id];
5786         struct instruction *ip = t->ip;
5787
5788         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5789
5790         JMP_CMP_MI(t, ip, >);
5791 }
5792
5793 static inline void
5794 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5795 {
5796         struct thread *t = &p->threads[p->thread_id];
5797         struct instruction *ip = t->ip;
5798
5799         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5800
5801         JMP_CMP_HI(t, ip, >);
5802 }
5803
5804 /*
5805  * return.
5806  */
5807 static int
5808 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5809                        struct action *action,
5810                        char **tokens __rte_unused,
5811                        int n_tokens,
5812                        struct instruction *instr,
5813                        struct instruction_data *data __rte_unused)
5814 {
5815         CHECK(action, EINVAL);
5816         CHECK(n_tokens == 1, EINVAL);
5817
5818         instr->type = INSTR_RETURN;
5819         return 0;
5820 }
5821
5822 static inline void
5823 instr_return_exec(struct rte_swx_pipeline *p)
5824 {
5825         struct thread *t = &p->threads[p->thread_id];
5826
5827         TRACE("[Thread %2u] return\n", p->thread_id);
5828
5829         t->ip = t->ret;
5830 }
5831
5832 static int
5833 instr_translate(struct rte_swx_pipeline *p,
5834                 struct action *action,
5835                 char *string,
5836                 struct instruction *instr,
5837                 struct instruction_data *data)
5838 {
5839         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5840         int n_tokens = 0, tpos = 0;
5841
5842         /* Parse the instruction string into tokens. */
5843         for ( ; ; ) {
5844                 char *token;
5845
5846                 token = strtok_r(string, " \t\v", &string);
5847                 if (!token)
5848                         break;
5849
5850                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5851                 CHECK_NAME(token, EINVAL);
5852
5853                 tokens[n_tokens] = token;
5854                 n_tokens++;
5855         }
5856
5857         CHECK(n_tokens, EINVAL);
5858
5859         /* Handle the optional instruction label. */
5860         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5861                 strcpy(data->label, tokens[0]);
5862
5863                 tpos += 2;
5864                 CHECK(n_tokens - tpos, EINVAL);
5865         }
5866
5867         /* Identify the instruction type. */
5868         if (!strcmp(tokens[tpos], "rx"))
5869                 return instr_rx_translate(p,
5870                                           action,
5871                                           &tokens[tpos],
5872                                           n_tokens - tpos,
5873                                           instr,
5874                                           data);
5875
5876         if (!strcmp(tokens[tpos], "tx"))
5877                 return instr_tx_translate(p,
5878                                           action,
5879                                           &tokens[tpos],
5880                                           n_tokens - tpos,
5881                                           instr,
5882                                           data);
5883
5884         if (!strcmp(tokens[tpos], "drop"))
5885                 return instr_drop_translate(p,
5886                                             action,
5887                                             &tokens[tpos],
5888                                             n_tokens - tpos,
5889                                             instr,
5890                                             data);
5891
5892         if (!strcmp(tokens[tpos], "extract"))
5893                 return instr_hdr_extract_translate(p,
5894                                                    action,
5895                                                    &tokens[tpos],
5896                                                    n_tokens - tpos,
5897                                                    instr,
5898                                                    data);
5899
5900         if (!strcmp(tokens[tpos], "lookahead"))
5901                 return instr_hdr_lookahead_translate(p,
5902                                                      action,
5903                                                      &tokens[tpos],
5904                                                      n_tokens - tpos,
5905                                                      instr,
5906                                                      data);
5907
5908         if (!strcmp(tokens[tpos], "emit"))
5909                 return instr_hdr_emit_translate(p,
5910                                                 action,
5911                                                 &tokens[tpos],
5912                                                 n_tokens - tpos,
5913                                                 instr,
5914                                                 data);
5915
5916         if (!strcmp(tokens[tpos], "validate"))
5917                 return instr_hdr_validate_translate(p,
5918                                                     action,
5919                                                     &tokens[tpos],
5920                                                     n_tokens - tpos,
5921                                                     instr,
5922                                                     data);
5923
5924         if (!strcmp(tokens[tpos], "invalidate"))
5925                 return instr_hdr_invalidate_translate(p,
5926                                                       action,
5927                                                       &tokens[tpos],
5928                                                       n_tokens - tpos,
5929                                                       instr,
5930                                                       data);
5931
5932         if (!strcmp(tokens[tpos], "mov"))
5933                 return instr_mov_translate(p,
5934                                            action,
5935                                            &tokens[tpos],
5936                                            n_tokens - tpos,
5937                                            instr,
5938                                            data);
5939
5940         if (!strcmp(tokens[tpos], "add"))
5941                 return instr_alu_add_translate(p,
5942                                                action,
5943                                                &tokens[tpos],
5944                                                n_tokens - tpos,
5945                                                instr,
5946                                                data);
5947
5948         if (!strcmp(tokens[tpos], "sub"))
5949                 return instr_alu_sub_translate(p,
5950                                                action,
5951                                                &tokens[tpos],
5952                                                n_tokens - tpos,
5953                                                instr,
5954                                                data);
5955
5956         if (!strcmp(tokens[tpos], "ckadd"))
5957                 return instr_alu_ckadd_translate(p,
5958                                                  action,
5959                                                  &tokens[tpos],
5960                                                  n_tokens - tpos,
5961                                                  instr,
5962                                                  data);
5963
5964         if (!strcmp(tokens[tpos], "cksub"))
5965                 return instr_alu_cksub_translate(p,
5966                                                  action,
5967                                                  &tokens[tpos],
5968                                                  n_tokens - tpos,
5969                                                  instr,
5970                                                  data);
5971
5972         if (!strcmp(tokens[tpos], "and"))
5973                 return instr_alu_and_translate(p,
5974                                                action,
5975                                                &tokens[tpos],
5976                                                n_tokens - tpos,
5977                                                instr,
5978                                                data);
5979
5980         if (!strcmp(tokens[tpos], "or"))
5981                 return instr_alu_or_translate(p,
5982                                               action,
5983                                               &tokens[tpos],
5984                                               n_tokens - tpos,
5985                                               instr,
5986                                               data);
5987
5988         if (!strcmp(tokens[tpos], "xor"))
5989                 return instr_alu_xor_translate(p,
5990                                                action,
5991                                                &tokens[tpos],
5992                                                n_tokens - tpos,
5993                                                instr,
5994                                                data);
5995
5996         if (!strcmp(tokens[tpos], "shl"))
5997                 return instr_alu_shl_translate(p,
5998                                                action,
5999                                                &tokens[tpos],
6000                                                n_tokens - tpos,
6001                                                instr,
6002                                                data);
6003
6004         if (!strcmp(tokens[tpos], "shr"))
6005                 return instr_alu_shr_translate(p,
6006                                                action,
6007                                                &tokens[tpos],
6008                                                n_tokens - tpos,
6009                                                instr,
6010                                                data);
6011
6012         if (!strcmp(tokens[tpos], "regprefetch"))
6013                 return instr_regprefetch_translate(p,
6014                                                    action,
6015                                                    &tokens[tpos],
6016                                                    n_tokens - tpos,
6017                                                    instr,
6018                                                    data);
6019
6020         if (!strcmp(tokens[tpos], "regrd"))
6021                 return instr_regrd_translate(p,
6022                                              action,
6023                                              &tokens[tpos],
6024                                              n_tokens - tpos,
6025                                              instr,
6026                                              data);
6027
6028         if (!strcmp(tokens[tpos], "regwr"))
6029                 return instr_regwr_translate(p,
6030                                              action,
6031                                              &tokens[tpos],
6032                                              n_tokens - tpos,
6033                                              instr,
6034                                              data);
6035
6036         if (!strcmp(tokens[tpos], "regadd"))
6037                 return instr_regadd_translate(p,
6038                                               action,
6039                                               &tokens[tpos],
6040                                               n_tokens - tpos,
6041                                               instr,
6042                                               data);
6043
6044         if (!strcmp(tokens[tpos], "metprefetch"))
6045                 return instr_metprefetch_translate(p,
6046                                                    action,
6047                                                    &tokens[tpos],
6048                                                    n_tokens - tpos,
6049                                                    instr,
6050                                                    data);
6051
6052         if (!strcmp(tokens[tpos], "meter"))
6053                 return instr_meter_translate(p,
6054                                              action,
6055                                              &tokens[tpos],
6056                                              n_tokens - tpos,
6057                                              instr,
6058                                              data);
6059
6060         if (!strcmp(tokens[tpos], "table"))
6061                 return instr_table_translate(p,
6062                                              action,
6063                                              &tokens[tpos],
6064                                              n_tokens - tpos,
6065                                              instr,
6066                                              data);
6067
6068         if (!strcmp(tokens[tpos], "learn"))
6069                 return instr_learn_translate(p,
6070                                              action,
6071                                              &tokens[tpos],
6072                                              n_tokens - tpos,
6073                                              instr,
6074                                              data);
6075
6076         if (!strcmp(tokens[tpos], "forget"))
6077                 return instr_forget_translate(p,
6078                                               action,
6079                                               &tokens[tpos],
6080                                               n_tokens - tpos,
6081                                               instr,
6082                                               data);
6083
6084         if (!strcmp(tokens[tpos], "extern"))
6085                 return instr_extern_translate(p,
6086                                               action,
6087                                               &tokens[tpos],
6088                                               n_tokens - tpos,
6089                                               instr,
6090                                               data);
6091
6092         if (!strcmp(tokens[tpos], "jmp"))
6093                 return instr_jmp_translate(p,
6094                                            action,
6095                                            &tokens[tpos],
6096                                            n_tokens - tpos,
6097                                            instr,
6098                                            data);
6099
6100         if (!strcmp(tokens[tpos], "jmpv"))
6101                 return instr_jmp_valid_translate(p,
6102                                                  action,
6103                                                  &tokens[tpos],
6104                                                  n_tokens - tpos,
6105                                                  instr,
6106                                                  data);
6107
6108         if (!strcmp(tokens[tpos], "jmpnv"))
6109                 return instr_jmp_invalid_translate(p,
6110                                                    action,
6111                                                    &tokens[tpos],
6112                                                    n_tokens - tpos,
6113                                                    instr,
6114                                                    data);
6115
6116         if (!strcmp(tokens[tpos], "jmph"))
6117                 return instr_jmp_hit_translate(p,
6118                                                action,
6119                                                &tokens[tpos],
6120                                                n_tokens - tpos,
6121                                                instr,
6122                                                data);
6123
6124         if (!strcmp(tokens[tpos], "jmpnh"))
6125                 return instr_jmp_miss_translate(p,
6126                                                 action,
6127                                                 &tokens[tpos],
6128                                                 n_tokens - tpos,
6129                                                 instr,
6130                                                 data);
6131
6132         if (!strcmp(tokens[tpos], "jmpa"))
6133                 return instr_jmp_action_hit_translate(p,
6134                                                       action,
6135                                                       &tokens[tpos],
6136                                                       n_tokens - tpos,
6137                                                       instr,
6138                                                       data);
6139
6140         if (!strcmp(tokens[tpos], "jmpna"))
6141                 return instr_jmp_action_miss_translate(p,
6142                                                        action,
6143                                                        &tokens[tpos],
6144                                                        n_tokens - tpos,
6145                                                        instr,
6146                                                        data);
6147
6148         if (!strcmp(tokens[tpos], "jmpeq"))
6149                 return instr_jmp_eq_translate(p,
6150                                               action,
6151                                               &tokens[tpos],
6152                                               n_tokens - tpos,
6153                                               instr,
6154                                               data);
6155
6156         if (!strcmp(tokens[tpos], "jmpneq"))
6157                 return instr_jmp_neq_translate(p,
6158                                                action,
6159                                                &tokens[tpos],
6160                                                n_tokens - tpos,
6161                                                instr,
6162                                                data);
6163
6164         if (!strcmp(tokens[tpos], "jmplt"))
6165                 return instr_jmp_lt_translate(p,
6166                                               action,
6167                                               &tokens[tpos],
6168                                               n_tokens - tpos,
6169                                               instr,
6170                                               data);
6171
6172         if (!strcmp(tokens[tpos], "jmpgt"))
6173                 return instr_jmp_gt_translate(p,
6174                                               action,
6175                                               &tokens[tpos],
6176                                               n_tokens - tpos,
6177                                               instr,
6178                                               data);
6179
6180         if (!strcmp(tokens[tpos], "return"))
6181                 return instr_return_translate(p,
6182                                               action,
6183                                               &tokens[tpos],
6184                                               n_tokens - tpos,
6185                                               instr,
6186                                               data);
6187
6188         CHECK(0, EINVAL);
6189 }
6190
6191 static struct instruction_data *
6192 label_find(struct instruction_data *data, uint32_t n, const char *label)
6193 {
6194         uint32_t i;
6195
6196         for (i = 0; i < n; i++)
6197                 if (!strcmp(label, data[i].label))
6198                         return &data[i];
6199
6200         return NULL;
6201 }
6202
6203 static uint32_t
6204 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6205 {
6206         uint32_t count = 0, i;
6207
6208         if (!label[0])
6209                 return 0;
6210
6211         for (i = 0; i < n; i++)
6212                 if (!strcmp(label, data[i].jmp_label))
6213                         count++;
6214
6215         return count;
6216 }
6217
6218 static int
6219 instr_label_check(struct instruction_data *instruction_data,
6220                   uint32_t n_instructions)
6221 {
6222         uint32_t i;
6223
6224         /* Check that all instruction labels are unique. */
6225         for (i = 0; i < n_instructions; i++) {
6226                 struct instruction_data *data = &instruction_data[i];
6227                 char *label = data->label;
6228                 uint32_t j;
6229
6230                 if (!label[0])
6231                         continue;
6232
6233                 for (j = i + 1; j < n_instructions; j++)
6234                         CHECK(strcmp(label, data[j].label), EINVAL);
6235         }
6236
6237         /* Get users for each instruction label. */
6238         for (i = 0; i < n_instructions; i++) {
6239                 struct instruction_data *data = &instruction_data[i];
6240                 char *label = data->label;
6241
6242                 data->n_users = label_is_used(instruction_data,
6243                                               n_instructions,
6244                                               label);
6245         }
6246
6247         return 0;
6248 }
6249
6250 static int
6251 instr_jmp_resolve(struct instruction *instructions,
6252                   struct instruction_data *instruction_data,
6253                   uint32_t n_instructions)
6254 {
6255         uint32_t i;
6256
6257         for (i = 0; i < n_instructions; i++) {
6258                 struct instruction *instr = &instructions[i];
6259                 struct instruction_data *data = &instruction_data[i];
6260                 struct instruction_data *found;
6261
6262                 if (!instruction_is_jmp(instr))
6263                         continue;
6264
6265                 found = label_find(instruction_data,
6266                                    n_instructions,
6267                                    data->jmp_label);
6268                 CHECK(found, EINVAL);
6269
6270                 instr->jmp.ip = &instructions[found - instruction_data];
6271         }
6272
6273         return 0;
6274 }
6275
6276 static int
6277 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6278              struct action *a,
6279              struct instruction *instr,
6280              struct instruction_data *data __rte_unused,
6281              uint32_t n_instructions)
6282 {
6283         if (!a) {
6284                 enum instruction_type type;
6285                 uint32_t i;
6286
6287                 /* Check that the first instruction is rx. */
6288                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6289
6290                 /* Check that there is at least one tx instruction. */
6291                 for (i = 0; i < n_instructions; i++) {
6292                         type = instr[i].type;
6293
6294                         if (instruction_is_tx(type))
6295                                 break;
6296                 }
6297                 CHECK(i < n_instructions, EINVAL);
6298
6299                 /* Check that the last instruction is either tx or unconditional
6300                  * jump.
6301                  */
6302                 type = instr[n_instructions - 1].type;
6303                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6304         }
6305
6306         if (a) {
6307                 enum instruction_type type;
6308                 uint32_t i;
6309
6310                 /* Check that there is at least one return or tx instruction. */
6311                 for (i = 0; i < n_instructions; i++) {
6312                         type = instr[i].type;
6313
6314                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6315                                 break;
6316                 }
6317                 CHECK(i < n_instructions, EINVAL);
6318         }
6319
6320         return 0;
6321 }
6322
6323 static uint32_t
6324 instr_compact(struct instruction *instructions,
6325               struct instruction_data *instruction_data,
6326               uint32_t n_instructions)
6327 {
6328         uint32_t i, pos = 0;
6329
6330         /* Eliminate the invalid instructions that have been optimized out. */
6331         for (i = 0; i < n_instructions; i++) {
6332                 struct instruction *instr = &instructions[i];
6333                 struct instruction_data *data = &instruction_data[i];
6334
6335                 if (data->invalid)
6336                         continue;
6337
6338                 if (i != pos) {
6339                         memcpy(&instructions[pos], instr, sizeof(*instr));
6340                         memcpy(&instruction_data[pos], data, sizeof(*data));
6341                 }
6342
6343                 pos++;
6344         }
6345
6346         return pos;
6347 }
6348
6349 static int
6350 instr_pattern_extract_many_search(struct instruction *instr,
6351                                   struct instruction_data *data,
6352                                   uint32_t n_instr,
6353                                   uint32_t *n_pattern_instr)
6354 {
6355         uint32_t i;
6356
6357         for (i = 0; i < n_instr; i++) {
6358                 if (data[i].invalid)
6359                         break;
6360
6361                 if (instr[i].type != INSTR_HDR_EXTRACT)
6362                         break;
6363
6364                 if (i == RTE_DIM(instr->io.hdr.header_id))
6365                         break;
6366
6367                 if (i && data[i].n_users)
6368                         break;
6369         }
6370
6371         if (i < 2)
6372                 return 0;
6373
6374         *n_pattern_instr = i;
6375         return 1;
6376 }
6377
6378 static void
6379 instr_pattern_extract_many_replace(struct instruction *instr,
6380                                    struct instruction_data *data,
6381                                    uint32_t n_instr)
6382 {
6383         uint32_t i;
6384
6385         for (i = 1; i < n_instr; i++) {
6386                 instr[0].type++;
6387                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6388                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6389                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6390
6391                 data[i].invalid = 1;
6392         }
6393 }
6394
6395 static uint32_t
6396 instr_pattern_extract_many_optimize(struct instruction *instructions,
6397                                     struct instruction_data *instruction_data,
6398                                     uint32_t n_instructions)
6399 {
6400         uint32_t i;
6401
6402         for (i = 0; i < n_instructions; ) {
6403                 struct instruction *instr = &instructions[i];
6404                 struct instruction_data *data = &instruction_data[i];
6405                 uint32_t n_instr = 0;
6406                 int detected;
6407
6408                 /* Extract many. */
6409                 detected = instr_pattern_extract_many_search(instr,
6410                                                              data,
6411                                                              n_instructions - i,
6412                                                              &n_instr);
6413                 if (detected) {
6414                         instr_pattern_extract_many_replace(instr,
6415                                                            data,
6416                                                            n_instr);
6417                         i += n_instr;
6418                         continue;
6419                 }
6420
6421                 /* No pattern starting at the current instruction. */
6422                 i++;
6423         }
6424
6425         /* Eliminate the invalid instructions that have been optimized out. */
6426         n_instructions = instr_compact(instructions,
6427                                        instruction_data,
6428                                        n_instructions);
6429
6430         return n_instructions;
6431 }
6432
6433 static int
6434 instr_pattern_emit_many_tx_search(struct instruction *instr,
6435                                   struct instruction_data *data,
6436                                   uint32_t n_instr,
6437                                   uint32_t *n_pattern_instr)
6438 {
6439         uint32_t i;
6440
6441         for (i = 0; i < n_instr; i++) {
6442                 if (data[i].invalid)
6443                         break;
6444
6445                 if (instr[i].type != INSTR_HDR_EMIT)
6446                         break;
6447
6448                 if (i == RTE_DIM(instr->io.hdr.header_id))
6449                         break;
6450
6451                 if (i && data[i].n_users)
6452                         break;
6453         }
6454
6455         if (!i)
6456                 return 0;
6457
6458         if (!instruction_is_tx(instr[i].type))
6459                 return 0;
6460
6461         if (data[i].n_users)
6462                 return 0;
6463
6464         i++;
6465
6466         *n_pattern_instr = i;
6467         return 1;
6468 }
6469
6470 static void
6471 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6472                                    struct instruction_data *data,
6473                                    uint32_t n_instr)
6474 {
6475         uint32_t i;
6476
6477         /* Any emit instruction in addition to the first one. */
6478         for (i = 1; i < n_instr - 1; i++) {
6479                 instr[0].type++;
6480                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6481                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6482                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6483
6484                 data[i].invalid = 1;
6485         }
6486
6487         /* The TX instruction is the last one in the pattern. */
6488         instr[0].type++;
6489         instr[0].io.io.offset = instr[i].io.io.offset;
6490         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6491         data[i].invalid = 1;
6492 }
6493
6494 static uint32_t
6495 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6496                                     struct instruction_data *instruction_data,
6497                                     uint32_t n_instructions)
6498 {
6499         uint32_t i;
6500
6501         for (i = 0; i < n_instructions; ) {
6502                 struct instruction *instr = &instructions[i];
6503                 struct instruction_data *data = &instruction_data[i];
6504                 uint32_t n_instr = 0;
6505                 int detected;
6506
6507                 /* Emit many + TX. */
6508                 detected = instr_pattern_emit_many_tx_search(instr,
6509                                                              data,
6510                                                              n_instructions - i,
6511                                                              &n_instr);
6512                 if (detected) {
6513                         instr_pattern_emit_many_tx_replace(instr,
6514                                                            data,
6515                                                            n_instr);
6516                         i += n_instr;
6517                         continue;
6518                 }
6519
6520                 /* No pattern starting at the current instruction. */
6521                 i++;
6522         }
6523
6524         /* Eliminate the invalid instructions that have been optimized out. */
6525         n_instructions = instr_compact(instructions,
6526                                        instruction_data,
6527                                        n_instructions);
6528
6529         return n_instructions;
6530 }
6531
6532 static uint32_t
6533 action_arg_src_mov_count(struct action *a,
6534                          uint32_t arg_id,
6535                          struct instruction *instructions,
6536                          struct instruction_data *instruction_data,
6537                          uint32_t n_instructions);
6538
6539 static int
6540 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6541                                       struct action *a,
6542                                       struct instruction *instr,
6543                                       struct instruction_data *data,
6544                                       uint32_t n_instr,
6545                                       struct instruction *instructions,
6546                                       struct instruction_data *instruction_data,
6547                                       uint32_t n_instructions,
6548                                       uint32_t *n_pattern_instr)
6549 {
6550         struct header *h;
6551         uint32_t src_field_id, i, j;
6552
6553         /* Prerequisites. */
6554         if (!a || !a->st)
6555                 return 0;
6556
6557         /* First instruction: MOV_HM. */
6558         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6559                 return 0;
6560
6561         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6562         if (!h || h->st->var_size)
6563                 return 0;
6564
6565         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6566                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6567                         break;
6568
6569         if (src_field_id == a->st->n_fields)
6570                 return 0;
6571
6572         if (instr[0].mov.dst.offset ||
6573             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6574             instr[0].mov.src.struct_id ||
6575             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6576             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6577                 return 0;
6578
6579         if ((n_instr < h->st->n_fields + 1) ||
6580              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6581                 return 0;
6582
6583         /* Subsequent instructions: MOV_HM. */
6584         for (i = 1; i < h->st->n_fields; i++)
6585                 if (data[i].invalid ||
6586                     data[i].n_users ||
6587                     (instr[i].type != INSTR_MOV_HM) ||
6588                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6589                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6590                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6591                     instr[i].mov.src.struct_id ||
6592                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6593                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6594                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6595                         return 0;
6596
6597         /* Last instruction: HDR_VALIDATE. */
6598         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6599             (instr[i].valid.header_id != h->id))
6600                 return 0;
6601
6602         /* Check that none of the action args that are used as source for this
6603          * DMA transfer are not used as source in any other mov instruction.
6604          */
6605         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6606                 uint32_t n_users;
6607
6608                 n_users = action_arg_src_mov_count(a,
6609                                                    j,
6610                                                    instructions,
6611                                                    instruction_data,
6612                                                    n_instructions);
6613                 if (n_users > 1)
6614                         return 0;
6615         }
6616
6617         *n_pattern_instr = 1 + i;
6618         return 1;
6619 }
6620
6621 static void
6622 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6623                                        struct action *a,
6624                                        struct instruction *instr,
6625                                        struct instruction_data *data,
6626                                        uint32_t n_instr)
6627 {
6628         struct header *h;
6629         uint32_t src_field_id, src_offset, i;
6630
6631         /* Read from the instructions before they are modified. */
6632         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6633         if (!h)
6634                 return;
6635
6636         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6637                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6638                         break;
6639
6640         if (src_field_id == a->st->n_fields)
6641                 return;
6642
6643         src_offset = instr[0].mov.src.offset;
6644
6645         /* Modify the instructions. */
6646         instr[0].type = INSTR_DMA_HT;
6647         instr[0].dma.dst.header_id[0] = h->id;
6648         instr[0].dma.dst.struct_id[0] = h->struct_id;
6649         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6650         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6651
6652         for (i = 1; i < n_instr; i++)
6653                 data[i].invalid = 1;
6654
6655         /* Update the endianness of the action arguments to header endianness. */
6656         for (i = 0; i < h->st->n_fields; i++)
6657                 a->args_endianness[src_field_id + i] = 1;
6658 }
6659
6660 static uint32_t
6661 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6662                                         struct action *a,
6663                                         struct instruction *instructions,
6664                                         struct instruction_data *instruction_data,
6665                                         uint32_t n_instructions)
6666 {
6667         uint32_t i;
6668
6669         if (!a || !a->st)
6670                 return n_instructions;
6671
6672         for (i = 0; i < n_instructions; ) {
6673                 struct instruction *instr = &instructions[i];
6674                 struct instruction_data *data = &instruction_data[i];
6675                 uint32_t n_instr = 0;
6676                 int detected;
6677
6678                 /* Mov all + validate. */
6679                 detected = instr_pattern_mov_all_validate_search(p,
6680                                                                  a,
6681                                                                  instr,
6682                                                                  data,
6683                                                                  n_instructions - i,
6684                                                                  instructions,
6685                                                                  instruction_data,
6686                                                                  n_instructions,
6687                                                                  &n_instr);
6688                 if (detected) {
6689                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6690                         i += n_instr;
6691                         continue;
6692                 }
6693
6694                 /* No pattern starting at the current instruction. */
6695                 i++;
6696         }
6697
6698         /* Eliminate the invalid instructions that have been optimized out. */
6699         n_instructions = instr_compact(instructions,
6700                                        instruction_data,
6701                                        n_instructions);
6702
6703         return n_instructions;
6704 }
6705
6706 static int
6707 instr_pattern_dma_many_search(struct instruction *instr,
6708                               struct instruction_data *data,
6709                               uint32_t n_instr,
6710                               uint32_t *n_pattern_instr)
6711 {
6712         uint32_t i;
6713
6714         for (i = 0; i < n_instr; i++) {
6715                 if (data[i].invalid)
6716                         break;
6717
6718                 if (instr[i].type != INSTR_DMA_HT)
6719                         break;
6720
6721                 if (i == RTE_DIM(instr->dma.dst.header_id))
6722                         break;
6723
6724                 if (i && data[i].n_users)
6725                         break;
6726         }
6727
6728         if (i < 2)
6729                 return 0;
6730
6731         *n_pattern_instr = i;
6732         return 1;
6733 }
6734
6735 static void
6736 instr_pattern_dma_many_replace(struct instruction *instr,
6737                                struct instruction_data *data,
6738                                uint32_t n_instr)
6739 {
6740         uint32_t i;
6741
6742         for (i = 1; i < n_instr; i++) {
6743                 instr[0].type++;
6744                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6745                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6746                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6747                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6748
6749                 data[i].invalid = 1;
6750         }
6751 }
6752
6753 static uint32_t
6754 instr_pattern_dma_many_optimize(struct instruction *instructions,
6755                struct instruction_data *instruction_data,
6756                uint32_t n_instructions)
6757 {
6758         uint32_t i;
6759
6760         for (i = 0; i < n_instructions; ) {
6761                 struct instruction *instr = &instructions[i];
6762                 struct instruction_data *data = &instruction_data[i];
6763                 uint32_t n_instr = 0;
6764                 int detected;
6765
6766                 /* DMA many. */
6767                 detected = instr_pattern_dma_many_search(instr,
6768                                                          data,
6769                                                          n_instructions - i,
6770                                                          &n_instr);
6771                 if (detected) {
6772                         instr_pattern_dma_many_replace(instr, data, n_instr);
6773                         i += n_instr;
6774                         continue;
6775                 }
6776
6777                 /* No pattern starting at the current instruction. */
6778                 i++;
6779         }
6780
6781         /* Eliminate the invalid instructions that have been optimized out. */
6782         n_instructions = instr_compact(instructions,
6783                                        instruction_data,
6784                                        n_instructions);
6785
6786         return n_instructions;
6787 }
6788
6789 static uint32_t
6790 instr_optimize(struct rte_swx_pipeline *p,
6791                struct action *a,
6792                struct instruction *instructions,
6793                struct instruction_data *instruction_data,
6794                uint32_t n_instructions)
6795 {
6796         /* Extract many. */
6797         n_instructions = instr_pattern_extract_many_optimize(instructions,
6798                                                              instruction_data,
6799                                                              n_instructions);
6800
6801         /* Emit many + TX. */
6802         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6803                                                              instruction_data,
6804                                                              n_instructions);
6805
6806         /* Mov all + validate. */
6807         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6808                                                                  a,
6809                                                                  instructions,
6810                                                                  instruction_data,
6811                                                                  n_instructions);
6812
6813         /* DMA many. */
6814         n_instructions = instr_pattern_dma_many_optimize(instructions,
6815                                                          instruction_data,
6816                                                          n_instructions);
6817
6818         return n_instructions;
6819 }
6820
6821 static int
6822 instruction_config(struct rte_swx_pipeline *p,
6823                    struct action *a,
6824                    const char **instructions,
6825                    uint32_t n_instructions)
6826 {
6827         struct instruction *instr = NULL;
6828         struct instruction_data *data = NULL;
6829         int err = 0;
6830         uint32_t i;
6831
6832         CHECK(n_instructions, EINVAL);
6833         CHECK(instructions, EINVAL);
6834         for (i = 0; i < n_instructions; i++)
6835                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6836
6837         /* Memory allocation. */
6838         instr = calloc(n_instructions, sizeof(struct instruction));
6839         if (!instr) {
6840                 err = -ENOMEM;
6841                 goto error;
6842         }
6843
6844         data = calloc(n_instructions, sizeof(struct instruction_data));
6845         if (!data) {
6846                 err = -ENOMEM;
6847                 goto error;
6848         }
6849
6850         for (i = 0; i < n_instructions; i++) {
6851                 char *string = strdup(instructions[i]);
6852                 if (!string) {
6853                         err = -ENOMEM;
6854                         goto error;
6855                 }
6856
6857                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6858                 if (err) {
6859                         free(string);
6860                         goto error;
6861                 }
6862
6863                 free(string);
6864         }
6865
6866         err = instr_label_check(data, n_instructions);
6867         if (err)
6868                 goto error;
6869
6870         err = instr_verify(p, a, instr, data, n_instructions);
6871         if (err)
6872                 goto error;
6873
6874         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6875
6876         err = instr_jmp_resolve(instr, data, n_instructions);
6877         if (err)
6878                 goto error;
6879
6880         if (a) {
6881                 a->instructions = instr;
6882                 a->n_instructions = n_instructions;
6883         } else {
6884                 p->instructions = instr;
6885                 p->n_instructions = n_instructions;
6886         }
6887
6888         free(data);
6889         return 0;
6890
6891 error:
6892         free(data);
6893         free(instr);
6894         return err;
6895 }
6896
6897 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
6898
6899 static instr_exec_t instruction_table[] = {
6900         [INSTR_RX] = instr_rx_exec,
6901         [INSTR_TX] = instr_tx_exec,
6902         [INSTR_TX_I] = instr_tx_i_exec,
6903
6904         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6905         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6906         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6907         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6908         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6909         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6910         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6911         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6912         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6913         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6914
6915         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6916         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6917         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6918         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6919         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6920         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6921         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6922         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6923         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6924
6925         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6926         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6927
6928         [INSTR_MOV] = instr_mov_exec,
6929         [INSTR_MOV_MH] = instr_mov_mh_exec,
6930         [INSTR_MOV_HM] = instr_mov_hm_exec,
6931         [INSTR_MOV_HH] = instr_mov_hh_exec,
6932         [INSTR_MOV_I] = instr_mov_i_exec,
6933
6934         [INSTR_DMA_HT] = instr_dma_ht_exec,
6935         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6936         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6937         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6938         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6939         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6940         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6941         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6942
6943         [INSTR_ALU_ADD] = instr_alu_add_exec,
6944         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6945         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6946         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6947         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6948         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6949
6950         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6951         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6952         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6953         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6954         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6955         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6956
6957         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6958         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6959         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6960         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6961
6962         [INSTR_ALU_AND] = instr_alu_and_exec,
6963         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6964         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6965         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6966         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6967
6968         [INSTR_ALU_OR] = instr_alu_or_exec,
6969         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6970         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6971         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6972         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6973
6974         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6975         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6976         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6977         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6978         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6979
6980         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6981         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6982         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6983         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6984         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6985         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6986
6987         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6988         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6989         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6990         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6991         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6992         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6993
6994         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6995         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6996         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6997
6998         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6999         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7000         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7001         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7002         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7003         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7004
7005         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7006         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7007         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7008         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7009         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7010         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7011         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7012         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7013         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7014
7015         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7016         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7017         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7018         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7019         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7020         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7021         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7022         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7023         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7024
7025         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7026         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7027         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7028
7029         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7030         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7031         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7032         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7033         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7034         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7035         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7036         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7037         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7038         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7039         [INSTR_METER_IMM] = instr_meter_imm_exec,
7040         [INSTR_METER_IMI] = instr_meter_imi_exec,
7041
7042         [INSTR_TABLE] = instr_table_exec,
7043         [INSTR_SELECTOR] = instr_selector_exec,
7044         [INSTR_LEARNER] = instr_learner_exec,
7045         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7046         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7047         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7048         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7049
7050         [INSTR_JMP] = instr_jmp_exec,
7051         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7052         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7053         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7054         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7055         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7056         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7057
7058         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7059         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7060         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7061         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7062         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7063
7064         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7065         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7066         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7067         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7068         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7069
7070         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7071         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7072         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7073         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7074         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7075         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7076
7077         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7078         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7079         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7080         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7081         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7082         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7083
7084         [INSTR_RETURN] = instr_return_exec,
7085 };
7086
7087 static inline void
7088 instr_exec(struct rte_swx_pipeline *p)
7089 {
7090         struct thread *t = &p->threads[p->thread_id];
7091         struct instruction *ip = t->ip;
7092         instr_exec_t instr = instruction_table[ip->type];
7093
7094         instr(p);
7095 }
7096
7097 /*
7098  * Action.
7099  */
7100 static struct action *
7101 action_find(struct rte_swx_pipeline *p, const char *name)
7102 {
7103         struct action *elem;
7104
7105         if (!name)
7106                 return NULL;
7107
7108         TAILQ_FOREACH(elem, &p->actions, node)
7109                 if (strcmp(elem->name, name) == 0)
7110                         return elem;
7111
7112         return NULL;
7113 }
7114
7115 static struct action *
7116 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7117 {
7118         struct action *action = NULL;
7119
7120         TAILQ_FOREACH(action, &p->actions, node)
7121                 if (action->id == id)
7122                         return action;
7123
7124         return NULL;
7125 }
7126
7127 static struct field *
7128 action_field_find(struct action *a, const char *name)
7129 {
7130         return a->st ? struct_type_field_find(a->st, name) : NULL;
7131 }
7132
7133 static struct field *
7134 action_field_parse(struct action *action, const char *name)
7135 {
7136         if (name[0] != 't' || name[1] != '.')
7137                 return NULL;
7138
7139         return action_field_find(action, &name[2]);
7140 }
7141
7142 static int
7143 action_has_nbo_args(struct action *a)
7144 {
7145         uint32_t i;
7146
7147         /* Return if the action does not have any args. */
7148         if (!a->st)
7149                 return 0; /* FALSE */
7150
7151         for (i = 0; i < a->st->n_fields; i++)
7152                 if (a->args_endianness[i])
7153                         return 1; /* TRUE */
7154
7155         return 0; /* FALSE */
7156 }
7157
7158 static int
7159 action_does_learning(struct action *a)
7160 {
7161         uint32_t i;
7162
7163         for (i = 0; i < a->n_instructions; i++)
7164                 switch (a->instructions[i].type) {
7165                 case INSTR_LEARNER_LEARN:
7166                         return 1; /* TRUE */
7167
7168                 case INSTR_LEARNER_FORGET:
7169                         return 1; /* TRUE */
7170
7171                 default:
7172                         continue;
7173                 }
7174
7175         return 0; /* FALSE */
7176 }
7177
7178 int
7179 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7180                                const char *name,
7181                                const char *args_struct_type_name,
7182                                const char **instructions,
7183                                uint32_t n_instructions)
7184 {
7185         struct struct_type *args_struct_type = NULL;
7186         struct action *a;
7187         int err;
7188
7189         CHECK(p, EINVAL);
7190
7191         CHECK_NAME(name, EINVAL);
7192         CHECK(!action_find(p, name), EEXIST);
7193
7194         if (args_struct_type_name) {
7195                 CHECK_NAME(args_struct_type_name, EINVAL);
7196                 args_struct_type = struct_type_find(p, args_struct_type_name);
7197                 CHECK(args_struct_type, EINVAL);
7198                 CHECK(!args_struct_type->var_size, EINVAL);
7199         }
7200
7201         /* Node allocation. */
7202         a = calloc(1, sizeof(struct action));
7203         CHECK(a, ENOMEM);
7204         if (args_struct_type) {
7205                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7206                 if (!a->args_endianness) {
7207                         free(a);
7208                         CHECK(0, ENOMEM);
7209                 }
7210         }
7211
7212         /* Node initialization. */
7213         strcpy(a->name, name);
7214         a->st = args_struct_type;
7215         a->id = p->n_actions;
7216
7217         /* Instruction translation. */
7218         err = instruction_config(p, a, instructions, n_instructions);
7219         if (err) {
7220                 free(a->args_endianness);
7221                 free(a);
7222                 return err;
7223         }
7224
7225         /* Node add to tailq. */
7226         TAILQ_INSERT_TAIL(&p->actions, a, node);
7227         p->n_actions++;
7228
7229         return 0;
7230 }
7231
7232 static int
7233 action_build(struct rte_swx_pipeline *p)
7234 {
7235         struct action *action;
7236
7237         p->action_instructions = calloc(p->n_actions,
7238                                         sizeof(struct instruction *));
7239         CHECK(p->action_instructions, ENOMEM);
7240
7241         TAILQ_FOREACH(action, &p->actions, node)
7242                 p->action_instructions[action->id] = action->instructions;
7243
7244         return 0;
7245 }
7246
7247 static void
7248 action_build_free(struct rte_swx_pipeline *p)
7249 {
7250         free(p->action_instructions);
7251         p->action_instructions = NULL;
7252 }
7253
7254 static void
7255 action_free(struct rte_swx_pipeline *p)
7256 {
7257         action_build_free(p);
7258
7259         for ( ; ; ) {
7260                 struct action *action;
7261
7262                 action = TAILQ_FIRST(&p->actions);
7263                 if (!action)
7264                         break;
7265
7266                 TAILQ_REMOVE(&p->actions, action, node);
7267                 free(action->instructions);
7268                 free(action);
7269         }
7270 }
7271
7272 static uint32_t
7273 action_arg_src_mov_count(struct action *a,
7274                          uint32_t arg_id,
7275                          struct instruction *instructions,
7276                          struct instruction_data *instruction_data,
7277                          uint32_t n_instructions)
7278 {
7279         uint32_t offset, n_users = 0, i;
7280
7281         if (!a->st ||
7282             (arg_id >= a->st->n_fields) ||
7283             !instructions ||
7284             !instruction_data ||
7285             !n_instructions)
7286                 return 0;
7287
7288         offset = a->st->fields[arg_id].offset / 8;
7289
7290         for (i = 0; i < n_instructions; i++) {
7291                 struct instruction *instr = &instructions[i];
7292                 struct instruction_data *data = &instruction_data[i];
7293
7294                 if (data->invalid ||
7295                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7296                     instr->mov.src.struct_id ||
7297                     (instr->mov.src.offset != offset))
7298                         continue;
7299
7300                 n_users++;
7301         }
7302
7303         return n_users;
7304 }
7305
7306 /*
7307  * Table.
7308  */
7309 static struct table_type *
7310 table_type_find(struct rte_swx_pipeline *p, const char *name)
7311 {
7312         struct table_type *elem;
7313
7314         TAILQ_FOREACH(elem, &p->table_types, node)
7315                 if (strcmp(elem->name, name) == 0)
7316                         return elem;
7317
7318         return NULL;
7319 }
7320
7321 static struct table_type *
7322 table_type_resolve(struct rte_swx_pipeline *p,
7323                    const char *recommended_type_name,
7324                    enum rte_swx_table_match_type match_type)
7325 {
7326         struct table_type *elem;
7327
7328         /* Only consider the recommended type if the match type is correct. */
7329         if (recommended_type_name)
7330                 TAILQ_FOREACH(elem, &p->table_types, node)
7331                         if (!strcmp(elem->name, recommended_type_name) &&
7332                             (elem->match_type == match_type))
7333                                 return elem;
7334
7335         /* Ignore the recommended type and get the first element with this match
7336          * type.
7337          */
7338         TAILQ_FOREACH(elem, &p->table_types, node)
7339                 if (elem->match_type == match_type)
7340                         return elem;
7341
7342         return NULL;
7343 }
7344
7345 static struct table *
7346 table_find(struct rte_swx_pipeline *p, const char *name)
7347 {
7348         struct table *elem;
7349
7350         TAILQ_FOREACH(elem, &p->tables, node)
7351                 if (strcmp(elem->name, name) == 0)
7352                         return elem;
7353
7354         return NULL;
7355 }
7356
7357 static struct table *
7358 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7359 {
7360         struct table *table = NULL;
7361
7362         TAILQ_FOREACH(table, &p->tables, node)
7363                 if (table->id == id)
7364                         return table;
7365
7366         return NULL;
7367 }
7368
7369 int
7370 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7371                                      const char *name,
7372                                      enum rte_swx_table_match_type match_type,
7373                                      struct rte_swx_table_ops *ops)
7374 {
7375         struct table_type *elem;
7376
7377         CHECK(p, EINVAL);
7378
7379         CHECK_NAME(name, EINVAL);
7380         CHECK(!table_type_find(p, name), EEXIST);
7381
7382         CHECK(ops, EINVAL);
7383         CHECK(ops->create, EINVAL);
7384         CHECK(ops->lkp, EINVAL);
7385         CHECK(ops->free, EINVAL);
7386
7387         /* Node allocation. */
7388         elem = calloc(1, sizeof(struct table_type));
7389         CHECK(elem, ENOMEM);
7390
7391         /* Node initialization. */
7392         strcpy(elem->name, name);
7393         elem->match_type = match_type;
7394         memcpy(&elem->ops, ops, sizeof(*ops));
7395
7396         /* Node add to tailq. */
7397         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7398
7399         return 0;
7400 }
7401
7402 static int
7403 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7404                          uint32_t n_fields,
7405                          enum rte_swx_table_match_type *match_type)
7406 {
7407         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7408
7409         for (i = 0; i < n_fields; i++) {
7410                 struct rte_swx_match_field_params  *f = &fields[i];
7411
7412                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7413                         n_fields_em++;
7414
7415                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7416                         n_fields_lpm++;
7417         }
7418
7419         if ((n_fields_lpm > 1) ||
7420             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7421                 return -EINVAL;
7422
7423         *match_type = (n_fields_em == n_fields) ?
7424                        RTE_SWX_TABLE_MATCH_EXACT :
7425                        RTE_SWX_TABLE_MATCH_WILDCARD;
7426
7427         return 0;
7428 }
7429
7430 static int
7431 table_match_fields_check(struct rte_swx_pipeline *p,
7432                          struct rte_swx_pipeline_table_params *params,
7433                          struct header **header)
7434 {
7435         struct header *h0 = NULL;
7436         struct field *hf, *mf;
7437         uint32_t *offset = NULL, i;
7438         int status = 0;
7439
7440         /* Return if no match fields. */
7441         if (!params->n_fields) {
7442                 if (params->fields) {
7443                         status = -EINVAL;
7444                         goto end;
7445                 }
7446
7447                 if (header)
7448                         *header = NULL;
7449
7450                 return 0;
7451         }
7452
7453         /* Memory allocation. */
7454         offset = calloc(params->n_fields, sizeof(uint32_t));
7455         if (!offset) {
7456                 status = -ENOMEM;
7457                 goto end;
7458         }
7459
7460         /* Check that all the match fields belong to either the same header or
7461          * to the meta-data.
7462          */
7463         hf = header_field_parse(p, params->fields[0].name, &h0);
7464         mf = metadata_field_parse(p, params->fields[0].name);
7465         if ((!hf && !mf) || (hf && hf->var_size)) {
7466                 status = -EINVAL;
7467                 goto end;
7468         }
7469
7470         offset[0] = h0 ? hf->offset : mf->offset;
7471
7472         for (i = 1; i < params->n_fields; i++)
7473                 if (h0) {
7474                         struct header *h;
7475
7476                         hf = header_field_parse(p, params->fields[i].name, &h);
7477                         if (!hf || (h->id != h0->id) || hf->var_size) {
7478                                 status = -EINVAL;
7479                                 goto end;
7480                         }
7481
7482                         offset[i] = hf->offset;
7483                 } else {
7484                         mf = metadata_field_parse(p, params->fields[i].name);
7485                         if (!mf) {
7486                                 status = -EINVAL;
7487                                 goto end;
7488                         }
7489
7490                         offset[i] = mf->offset;
7491                 }
7492
7493         /* Check that there are no duplicated match fields. */
7494         for (i = 0; i < params->n_fields; i++) {
7495                 uint32_t j;
7496
7497                 for (j = 0; j < i; j++)
7498                         if (offset[j] == offset[i]) {
7499                                 status = -EINVAL;
7500                                 goto end;
7501                         }
7502         }
7503
7504         /* Return. */
7505         if (header)
7506                 *header = h0;
7507
7508 end:
7509         free(offset);
7510         return status;
7511 }
7512
7513 int
7514 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7515                               const char *name,
7516                               struct rte_swx_pipeline_table_params *params,
7517                               const char *recommended_table_type_name,
7518                               const char *args,
7519                               uint32_t size)
7520 {
7521         struct table_type *type;
7522         struct table *t;
7523         struct action *default_action;
7524         struct header *header = NULL;
7525         uint32_t action_data_size_max = 0, i;
7526         int status = 0;
7527
7528         CHECK(p, EINVAL);
7529
7530         CHECK_NAME(name, EINVAL);
7531         CHECK(!table_find(p, name), EEXIST);
7532         CHECK(!selector_find(p, name), EEXIST);
7533         CHECK(!learner_find(p, name), EEXIST);
7534
7535         CHECK(params, EINVAL);
7536
7537         /* Match checks. */
7538         status = table_match_fields_check(p, params, &header);
7539         if (status)
7540                 return status;
7541
7542         /* Action checks. */
7543         CHECK(params->n_actions, EINVAL);
7544         CHECK(params->action_names, EINVAL);
7545         for (i = 0; i < params->n_actions; i++) {
7546                 const char *action_name = params->action_names[i];
7547                 struct action *a;
7548                 uint32_t action_data_size;
7549
7550                 CHECK_NAME(action_name, EINVAL);
7551
7552                 a = action_find(p, action_name);
7553                 CHECK(a, EINVAL);
7554                 CHECK(!action_does_learning(a), EINVAL);
7555
7556                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7557                 if (action_data_size > action_data_size_max)
7558                         action_data_size_max = action_data_size;
7559         }
7560
7561         CHECK_NAME(params->default_action_name, EINVAL);
7562         for (i = 0; i < p->n_actions; i++)
7563                 if (!strcmp(params->action_names[i],
7564                             params->default_action_name))
7565                         break;
7566         CHECK(i < params->n_actions, EINVAL);
7567         default_action = action_find(p, params->default_action_name);
7568         CHECK((default_action->st && params->default_action_data) ||
7569               !params->default_action_data, EINVAL);
7570
7571         /* Table type checks. */
7572         if (recommended_table_type_name)
7573                 CHECK_NAME(recommended_table_type_name, EINVAL);
7574
7575         if (params->n_fields) {
7576                 enum rte_swx_table_match_type match_type;
7577
7578                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7579                 if (status)
7580                         return status;
7581
7582                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7583                 CHECK(type, EINVAL);
7584         } else {
7585                 type = NULL;
7586         }
7587
7588         /* Memory allocation. */
7589         t = calloc(1, sizeof(struct table));
7590         CHECK(t, ENOMEM);
7591
7592         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7593         if (!t->fields) {
7594                 free(t);
7595                 CHECK(0, ENOMEM);
7596         }
7597
7598         t->actions = calloc(params->n_actions, sizeof(struct action *));
7599         if (!t->actions) {
7600                 free(t->fields);
7601                 free(t);
7602                 CHECK(0, ENOMEM);
7603         }
7604
7605         if (action_data_size_max) {
7606                 t->default_action_data = calloc(1, action_data_size_max);
7607                 if (!t->default_action_data) {
7608                         free(t->actions);
7609                         free(t->fields);
7610                         free(t);
7611                         CHECK(0, ENOMEM);
7612                 }
7613         }
7614
7615         /* Node initialization. */
7616         strcpy(t->name, name);
7617         if (args && args[0])
7618                 strcpy(t->args, args);
7619         t->type = type;
7620
7621         for (i = 0; i < params->n_fields; i++) {
7622                 struct rte_swx_match_field_params *field = &params->fields[i];
7623                 struct match_field *f = &t->fields[i];
7624
7625                 f->match_type = field->match_type;
7626                 f->field = header ?
7627                         header_field_parse(p, field->name, NULL) :
7628                         metadata_field_parse(p, field->name);
7629         }
7630         t->n_fields = params->n_fields;
7631         t->header = header;
7632
7633         for (i = 0; i < params->n_actions; i++)
7634                 t->actions[i] = action_find(p, params->action_names[i]);
7635         t->default_action = default_action;
7636         if (default_action->st)
7637                 memcpy(t->default_action_data,
7638                        params->default_action_data,
7639                        default_action->st->n_bits / 8);
7640         t->n_actions = params->n_actions;
7641         t->default_action_is_const = params->default_action_is_const;
7642         t->action_data_size_max = action_data_size_max;
7643
7644         t->size = size;
7645         t->id = p->n_tables;
7646
7647         /* Node add to tailq. */
7648         TAILQ_INSERT_TAIL(&p->tables, t, node);
7649         p->n_tables++;
7650
7651         return 0;
7652 }
7653
7654 static struct rte_swx_table_params *
7655 table_params_get(struct table *table)
7656 {
7657         struct rte_swx_table_params *params;
7658         struct field *first, *last;
7659         uint8_t *key_mask;
7660         uint32_t key_size, key_offset, action_data_size, i;
7661
7662         /* Memory allocation. */
7663         params = calloc(1, sizeof(struct rte_swx_table_params));
7664         if (!params)
7665                 return NULL;
7666
7667         /* Find first (smallest offset) and last (biggest offset) match fields. */
7668         first = table->fields[0].field;
7669         last = table->fields[0].field;
7670
7671         for (i = 0; i < table->n_fields; i++) {
7672                 struct field *f = table->fields[i].field;
7673
7674                 if (f->offset < first->offset)
7675                         first = f;
7676
7677                 if (f->offset > last->offset)
7678                         last = f;
7679         }
7680
7681         /* Key offset and size. */
7682         key_offset = first->offset / 8;
7683         key_size = (last->offset + last->n_bits - first->offset) / 8;
7684
7685         /* Memory allocation. */
7686         key_mask = calloc(1, key_size);
7687         if (!key_mask) {
7688                 free(params);
7689                 return NULL;
7690         }
7691
7692         /* Key mask. */
7693         for (i = 0; i < table->n_fields; i++) {
7694                 struct field *f = table->fields[i].field;
7695                 uint32_t start = (f->offset - first->offset) / 8;
7696                 size_t size = f->n_bits / 8;
7697
7698                 memset(&key_mask[start], 0xFF, size);
7699         }
7700
7701         /* Action data size. */
7702         action_data_size = 0;
7703         for (i = 0; i < table->n_actions; i++) {
7704                 struct action *action = table->actions[i];
7705                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7706
7707                 if (ads > action_data_size)
7708                         action_data_size = ads;
7709         }
7710
7711         /* Fill in. */
7712         params->match_type = table->type->match_type;
7713         params->key_size = key_size;
7714         params->key_offset = key_offset;
7715         params->key_mask0 = key_mask;
7716         params->action_data_size = action_data_size;
7717         params->n_keys_max = table->size;
7718
7719         return params;
7720 }
7721
7722 static void
7723 table_params_free(struct rte_swx_table_params *params)
7724 {
7725         if (!params)
7726                 return;
7727
7728         free(params->key_mask0);
7729         free(params);
7730 }
7731
7732 static int
7733 table_stub_lkp(void *table __rte_unused,
7734                void *mailbox __rte_unused,
7735                uint8_t **key __rte_unused,
7736                uint64_t *action_id __rte_unused,
7737                uint8_t **action_data __rte_unused,
7738                int *hit)
7739 {
7740         *hit = 0;
7741         return 1; /* DONE. */
7742 }
7743
7744 static int
7745 table_build(struct rte_swx_pipeline *p)
7746 {
7747         uint32_t i;
7748
7749         /* Per pipeline: table statistics. */
7750         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7751         CHECK(p->table_stats, ENOMEM);
7752
7753         for (i = 0; i < p->n_tables; i++) {
7754                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7755                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7756         }
7757
7758         /* Per thread: table runt-time. */
7759         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7760                 struct thread *t = &p->threads[i];
7761                 struct table *table;
7762
7763                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7764                 CHECK(t->tables, ENOMEM);
7765
7766                 TAILQ_FOREACH(table, &p->tables, node) {
7767                         struct table_runtime *r = &t->tables[table->id];
7768
7769                         if (table->type) {
7770                                 uint64_t size;
7771
7772                                 size = table->type->ops.mailbox_size_get();
7773
7774                                 /* r->func. */
7775                                 r->func = table->type->ops.lkp;
7776
7777                                 /* r->mailbox. */
7778                                 if (size) {
7779                                         r->mailbox = calloc(1, size);
7780                                         CHECK(r->mailbox, ENOMEM);
7781                                 }
7782
7783                                 /* r->key. */
7784                                 r->key = table->header ?
7785                                         &t->structs[table->header->struct_id] :
7786                                         &t->structs[p->metadata_struct_id];
7787                         } else {
7788                                 r->func = table_stub_lkp;
7789                         }
7790                 }
7791         }
7792
7793         return 0;
7794 }
7795
7796 static void
7797 table_build_free(struct rte_swx_pipeline *p)
7798 {
7799         uint32_t i;
7800
7801         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7802                 struct thread *t = &p->threads[i];
7803                 uint32_t j;
7804
7805                 if (!t->tables)
7806                         continue;
7807
7808                 for (j = 0; j < p->n_tables; j++) {
7809                         struct table_runtime *r = &t->tables[j];
7810
7811                         free(r->mailbox);
7812                 }
7813
7814                 free(t->tables);
7815                 t->tables = NULL;
7816         }
7817
7818         if (p->table_stats) {
7819                 for (i = 0; i < p->n_tables; i++)
7820                         free(p->table_stats[i].n_pkts_action);
7821
7822                 free(p->table_stats);
7823         }
7824 }
7825
7826 static void
7827 table_free(struct rte_swx_pipeline *p)
7828 {
7829         table_build_free(p);
7830
7831         /* Tables. */
7832         for ( ; ; ) {
7833                 struct table *elem;
7834
7835                 elem = TAILQ_FIRST(&p->tables);
7836                 if (!elem)
7837                         break;
7838
7839                 TAILQ_REMOVE(&p->tables, elem, node);
7840                 free(elem->fields);
7841                 free(elem->actions);
7842                 free(elem->default_action_data);
7843                 free(elem);
7844         }
7845
7846         /* Table types. */
7847         for ( ; ; ) {
7848                 struct table_type *elem;
7849
7850                 elem = TAILQ_FIRST(&p->table_types);
7851                 if (!elem)
7852                         break;
7853
7854                 TAILQ_REMOVE(&p->table_types, elem, node);
7855                 free(elem);
7856         }
7857 }
7858
7859 /*
7860  * Selector.
7861  */
7862 static struct selector *
7863 selector_find(struct rte_swx_pipeline *p, const char *name)
7864 {
7865         struct selector *s;
7866
7867         TAILQ_FOREACH(s, &p->selectors, node)
7868                 if (strcmp(s->name, name) == 0)
7869                         return s;
7870
7871         return NULL;
7872 }
7873
7874 static struct selector *
7875 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7876 {
7877         struct selector *s = NULL;
7878
7879         TAILQ_FOREACH(s, &p->selectors, node)
7880                 if (s->id == id)
7881                         return s;
7882
7883         return NULL;
7884 }
7885
7886 static int
7887 selector_fields_check(struct rte_swx_pipeline *p,
7888                       struct rte_swx_pipeline_selector_params *params,
7889                       struct header **header)
7890 {
7891         struct header *h0 = NULL;
7892         struct field *hf, *mf;
7893         uint32_t i;
7894
7895         /* Return if no selector fields. */
7896         if (!params->n_selector_fields || !params->selector_field_names)
7897                 return -EINVAL;
7898
7899         /* Check that all the selector fields either belong to the same header
7900          * or are all meta-data fields.
7901          */
7902         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7903         mf = metadata_field_parse(p, params->selector_field_names[0]);
7904         if (!hf && !mf)
7905                 return -EINVAL;
7906
7907         for (i = 1; i < params->n_selector_fields; i++)
7908                 if (h0) {
7909                         struct header *h;
7910
7911                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7912                         if (!hf || (h->id != h0->id))
7913                                 return -EINVAL;
7914                 } else {
7915                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7916                         if (!mf)
7917                                 return -EINVAL;
7918                 }
7919
7920         /* Check that there are no duplicated match fields. */
7921         for (i = 0; i < params->n_selector_fields; i++) {
7922                 const char *field_name = params->selector_field_names[i];
7923                 uint32_t j;
7924
7925                 for (j = i + 1; j < params->n_selector_fields; j++)
7926                         if (!strcmp(params->selector_field_names[j], field_name))
7927                                 return -EINVAL;
7928         }
7929
7930         /* Return. */
7931         if (header)
7932                 *header = h0;
7933
7934         return 0;
7935 }
7936
7937 int
7938 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7939                                  const char *name,
7940                                  struct rte_swx_pipeline_selector_params *params)
7941 {
7942         struct selector *s;
7943         struct header *selector_header = NULL;
7944         struct field *group_id_field, *member_id_field;
7945         uint32_t i;
7946         int status = 0;
7947
7948         CHECK(p, EINVAL);
7949
7950         CHECK_NAME(name, EINVAL);
7951         CHECK(!table_find(p, name), EEXIST);
7952         CHECK(!selector_find(p, name), EEXIST);
7953         CHECK(!learner_find(p, name), EEXIST);
7954
7955         CHECK(params, EINVAL);
7956
7957         CHECK_NAME(params->group_id_field_name, EINVAL);
7958         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7959         CHECK(group_id_field, EINVAL);
7960
7961         for (i = 0; i < params->n_selector_fields; i++) {
7962                 const char *field_name = params->selector_field_names[i];
7963
7964                 CHECK_NAME(field_name, EINVAL);
7965         }
7966         status = selector_fields_check(p, params, &selector_header);
7967         if (status)
7968                 return status;
7969
7970         CHECK_NAME(params->member_id_field_name, EINVAL);
7971         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7972         CHECK(member_id_field, EINVAL);
7973
7974         CHECK(params->n_groups_max, EINVAL);
7975
7976         CHECK(params->n_members_per_group_max, EINVAL);
7977
7978         /* Memory allocation. */
7979         s = calloc(1, sizeof(struct selector));
7980         if (!s) {
7981                 status = -ENOMEM;
7982                 goto error;
7983         }
7984
7985         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7986         if (!s->selector_fields) {
7987                 status = -ENOMEM;
7988                 goto error;
7989         }
7990
7991         /* Node initialization. */
7992         strcpy(s->name, name);
7993
7994         s->group_id_field = group_id_field;
7995
7996         for (i = 0; i < params->n_selector_fields; i++) {
7997                 const char *field_name = params->selector_field_names[i];
7998
7999                 s->selector_fields[i] = selector_header ?
8000                         header_field_parse(p, field_name, NULL) :
8001                         metadata_field_parse(p, field_name);
8002         }
8003
8004         s->n_selector_fields = params->n_selector_fields;
8005
8006         s->selector_header = selector_header;
8007
8008         s->member_id_field = member_id_field;
8009
8010         s->n_groups_max = params->n_groups_max;
8011
8012         s->n_members_per_group_max = params->n_members_per_group_max;
8013
8014         s->id = p->n_selectors;
8015
8016         /* Node add to tailq. */
8017         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8018         p->n_selectors++;
8019
8020         return 0;
8021
8022 error:
8023         if (!s)
8024                 return status;
8025
8026         free(s->selector_fields);
8027
8028         free(s);
8029
8030         return status;
8031 }
8032
8033 static void
8034 selector_params_free(struct rte_swx_table_selector_params *params)
8035 {
8036         if (!params)
8037                 return;
8038
8039         free(params->selector_mask);
8040
8041         free(params);
8042 }
8043
8044 static struct rte_swx_table_selector_params *
8045 selector_table_params_get(struct selector *s)
8046 {
8047         struct rte_swx_table_selector_params *params = NULL;
8048         struct field *first, *last;
8049         uint32_t i;
8050
8051         /* Memory allocation. */
8052         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8053         if (!params)
8054                 goto error;
8055
8056         /* Group ID. */
8057         params->group_id_offset = s->group_id_field->offset / 8;
8058
8059         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8060         first = s->selector_fields[0];
8061         last = s->selector_fields[0];
8062
8063         for (i = 0; i < s->n_selector_fields; i++) {
8064                 struct field *f = s->selector_fields[i];
8065
8066                 if (f->offset < first->offset)
8067                         first = f;
8068
8069                 if (f->offset > last->offset)
8070                         last = f;
8071         }
8072
8073         /* Selector offset and size. */
8074         params->selector_offset = first->offset / 8;
8075         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8076
8077         /* Memory allocation. */
8078         params->selector_mask = calloc(1, params->selector_size);
8079         if (!params->selector_mask)
8080                 goto error;
8081
8082         /* Selector mask. */
8083         for (i = 0; i < s->n_selector_fields; i++) {
8084                 struct field *f = s->selector_fields[i];
8085                 uint32_t start = (f->offset - first->offset) / 8;
8086                 size_t size = f->n_bits / 8;
8087
8088                 memset(&params->selector_mask[start], 0xFF, size);
8089         }
8090
8091         /* Member ID. */
8092         params->member_id_offset = s->member_id_field->offset / 8;
8093
8094         /* Maximum number of groups. */
8095         params->n_groups_max = s->n_groups_max;
8096
8097         /* Maximum number of members per group. */
8098         params->n_members_per_group_max = s->n_members_per_group_max;
8099
8100         return params;
8101
8102 error:
8103         selector_params_free(params);
8104         return NULL;
8105 }
8106
8107 static void
8108 selector_build_free(struct rte_swx_pipeline *p)
8109 {
8110         uint32_t i;
8111
8112         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8113                 struct thread *t = &p->threads[i];
8114                 uint32_t j;
8115
8116                 if (!t->selectors)
8117                         continue;
8118
8119                 for (j = 0; j < p->n_selectors; j++) {
8120                         struct selector_runtime *r = &t->selectors[j];
8121
8122                         free(r->mailbox);
8123                 }
8124
8125                 free(t->selectors);
8126                 t->selectors = NULL;
8127         }
8128
8129         free(p->selector_stats);
8130         p->selector_stats = NULL;
8131 }
8132
8133 static int
8134 selector_build(struct rte_swx_pipeline *p)
8135 {
8136         uint32_t i;
8137         int status = 0;
8138
8139         /* Per pipeline: selector statistics. */
8140         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8141         if (!p->selector_stats) {
8142                 status = -ENOMEM;
8143                 goto error;
8144         }
8145
8146         /* Per thread: selector run-time. */
8147         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8148                 struct thread *t = &p->threads[i];
8149                 struct selector *s;
8150
8151                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8152                 if (!t->selectors) {
8153                         status = -ENOMEM;
8154                         goto error;
8155                 }
8156
8157                 TAILQ_FOREACH(s, &p->selectors, node) {
8158                         struct selector_runtime *r = &t->selectors[s->id];
8159                         uint64_t size;
8160
8161                         /* r->mailbox. */
8162                         size = rte_swx_table_selector_mailbox_size_get();
8163                         if (size) {
8164                                 r->mailbox = calloc(1, size);
8165                                 if (!r->mailbox) {
8166                                         status = -ENOMEM;
8167                                         goto error;
8168                                 }
8169                         }
8170
8171                         /* r->group_id_buffer. */
8172                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8173
8174                         /* r->selector_buffer. */
8175                         r->selector_buffer = s->selector_header ?
8176                                 &t->structs[s->selector_header->struct_id] :
8177                                 &t->structs[p->metadata_struct_id];
8178
8179                         /* r->member_id_buffer. */
8180                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8181                 }
8182         }
8183
8184         return 0;
8185
8186 error:
8187         selector_build_free(p);
8188         return status;
8189 }
8190
8191 static void
8192 selector_free(struct rte_swx_pipeline *p)
8193 {
8194         selector_build_free(p);
8195
8196         /* Selector tables. */
8197         for ( ; ; ) {
8198                 struct selector *elem;
8199
8200                 elem = TAILQ_FIRST(&p->selectors);
8201                 if (!elem)
8202                         break;
8203
8204                 TAILQ_REMOVE(&p->selectors, elem, node);
8205                 free(elem->selector_fields);
8206                 free(elem);
8207         }
8208 }
8209
8210 /*
8211  * Learner table.
8212  */
8213 static struct learner *
8214 learner_find(struct rte_swx_pipeline *p, const char *name)
8215 {
8216         struct learner *l;
8217
8218         TAILQ_FOREACH(l, &p->learners, node)
8219                 if (!strcmp(l->name, name))
8220                         return l;
8221
8222         return NULL;
8223 }
8224
8225 static struct learner *
8226 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8227 {
8228         struct learner *l = NULL;
8229
8230         TAILQ_FOREACH(l, &p->learners, node)
8231                 if (l->id == id)
8232                         return l;
8233
8234         return NULL;
8235 }
8236
8237 static int
8238 learner_match_fields_check(struct rte_swx_pipeline *p,
8239                            struct rte_swx_pipeline_learner_params *params,
8240                            struct header **header)
8241 {
8242         struct header *h0 = NULL;
8243         struct field *hf, *mf;
8244         uint32_t i;
8245
8246         /* Return if no match fields. */
8247         if (!params->n_fields || !params->field_names)
8248                 return -EINVAL;
8249
8250         /* Check that all the match fields either belong to the same header
8251          * or are all meta-data fields.
8252          */
8253         hf = header_field_parse(p, params->field_names[0], &h0);
8254         mf = metadata_field_parse(p, params->field_names[0]);
8255         if (!hf && !mf)
8256                 return -EINVAL;
8257
8258         for (i = 1; i < params->n_fields; i++)
8259                 if (h0) {
8260                         struct header *h;
8261
8262                         hf = header_field_parse(p, params->field_names[i], &h);
8263                         if (!hf || (h->id != h0->id))
8264                                 return -EINVAL;
8265                 } else {
8266                         mf = metadata_field_parse(p, params->field_names[i]);
8267                         if (!mf)
8268                                 return -EINVAL;
8269                 }
8270
8271         /* Check that there are no duplicated match fields. */
8272         for (i = 0; i < params->n_fields; i++) {
8273                 const char *field_name = params->field_names[i];
8274                 uint32_t j;
8275
8276                 for (j = i + 1; j < params->n_fields; j++)
8277                         if (!strcmp(params->field_names[j], field_name))
8278                                 return -EINVAL;
8279         }
8280
8281         /* Return. */
8282         if (header)
8283                 *header = h0;
8284
8285         return 0;
8286 }
8287
8288 static int
8289 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8290 {
8291         struct struct_type *mst = p->metadata_st, *ast = a->st;
8292         struct field *mf, *af;
8293         uint32_t mf_pos, i;
8294
8295         if (!ast) {
8296                 if (mf_name)
8297                         return -EINVAL;
8298
8299                 return 0;
8300         }
8301
8302         /* Check that mf_name is the name of a valid meta-data field. */
8303         CHECK_NAME(mf_name, EINVAL);
8304         mf = metadata_field_parse(p, mf_name);
8305         CHECK(mf, EINVAL);
8306
8307         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8308          * all the action arguments.
8309          */
8310         mf_pos = mf - mst->fields;
8311         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8312
8313         /* Check that the size of each of the identified meta-data fields matches exactly the size
8314          * of the corresponding action argument.
8315          */
8316         for (i = 0; i < ast->n_fields; i++) {
8317                 mf = &mst->fields[mf_pos + i];
8318                 af = &ast->fields[i];
8319
8320                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8321         }
8322
8323         return 0;
8324 }
8325
8326 static int
8327 learner_action_learning_check(struct rte_swx_pipeline *p,
8328                               struct action *action,
8329                               const char **action_names,
8330                               uint32_t n_actions)
8331 {
8332         uint32_t i;
8333
8334         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8335          * the action passed as argument to the "learn" instruction) is also enabled for the
8336          * current learner table.
8337          */
8338         for (i = 0; i < action->n_instructions; i++) {
8339                 struct instruction *instr = &action->instructions[i];
8340                 uint32_t found = 0, j;
8341
8342                 if (instr->type != INSTR_LEARNER_LEARN)
8343                         continue;
8344
8345                 for (j = 0; j < n_actions; j++) {
8346                         struct action *a;
8347
8348                         a = action_find(p, action_names[j]);
8349                         if (!a)
8350                                 return -EINVAL;
8351
8352                         if (a->id == instr->learn.action_id)
8353                                 found = 1;
8354                 }
8355
8356                 if (!found)
8357                         return -EINVAL;
8358         }
8359
8360         return 0;
8361 }
8362
8363 int
8364 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8365                               const char *name,
8366                               struct rte_swx_pipeline_learner_params *params,
8367                               uint32_t size,
8368                               uint32_t timeout)
8369 {
8370         struct learner *l = NULL;
8371         struct action *default_action;
8372         struct header *header = NULL;
8373         uint32_t action_data_size_max = 0, i;
8374         int status = 0;
8375
8376         CHECK(p, EINVAL);
8377
8378         CHECK_NAME(name, EINVAL);
8379         CHECK(!table_find(p, name), EEXIST);
8380         CHECK(!selector_find(p, name), EEXIST);
8381         CHECK(!learner_find(p, name), EEXIST);
8382
8383         CHECK(params, EINVAL);
8384
8385         /* Match checks. */
8386         status = learner_match_fields_check(p, params, &header);
8387         if (status)
8388                 return status;
8389
8390         /* Action checks. */
8391         CHECK(params->n_actions, EINVAL);
8392
8393         CHECK(params->action_names, EINVAL);
8394         for (i = 0; i < params->n_actions; i++) {
8395                 const char *action_name = params->action_names[i];
8396                 const char *action_field_name = params->action_field_names[i];
8397                 struct action *a;
8398                 uint32_t action_data_size;
8399
8400                 CHECK_NAME(action_name, EINVAL);
8401
8402                 a = action_find(p, action_name);
8403                 CHECK(a, EINVAL);
8404
8405                 status = learner_action_args_check(p, a, action_field_name);
8406                 if (status)
8407                         return status;
8408
8409                 status = learner_action_learning_check(p,
8410                                                        a,
8411                                                        params->action_names,
8412                                                        params->n_actions);
8413                 if (status)
8414                         return status;
8415
8416                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8417                 if (action_data_size > action_data_size_max)
8418                         action_data_size_max = action_data_size;
8419         }
8420
8421         CHECK_NAME(params->default_action_name, EINVAL);
8422         for (i = 0; i < p->n_actions; i++)
8423                 if (!strcmp(params->action_names[i],
8424                             params->default_action_name))
8425                         break;
8426         CHECK(i < params->n_actions, EINVAL);
8427
8428         default_action = action_find(p, params->default_action_name);
8429         CHECK((default_action->st && params->default_action_data) ||
8430               !params->default_action_data, EINVAL);
8431
8432         /* Any other checks. */
8433         CHECK(size, EINVAL);
8434         CHECK(timeout, EINVAL);
8435
8436         /* Memory allocation. */
8437         l = calloc(1, sizeof(struct learner));
8438         if (!l)
8439                 goto nomem;
8440
8441         l->fields = calloc(params->n_fields, sizeof(struct field *));
8442         if (!l->fields)
8443                 goto nomem;
8444
8445         l->actions = calloc(params->n_actions, sizeof(struct action *));
8446         if (!l->actions)
8447                 goto nomem;
8448
8449         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8450         if (!l->action_arg)
8451                 goto nomem;
8452
8453         if (action_data_size_max) {
8454                 l->default_action_data = calloc(1, action_data_size_max);
8455                 if (!l->default_action_data)
8456                         goto nomem;
8457         }
8458
8459         /* Node initialization. */
8460         strcpy(l->name, name);
8461
8462         for (i = 0; i < params->n_fields; i++) {
8463                 const char *field_name = params->field_names[i];
8464
8465                 l->fields[i] = header ?
8466                         header_field_parse(p, field_name, NULL) :
8467                         metadata_field_parse(p, field_name);
8468         }
8469
8470         l->n_fields = params->n_fields;
8471
8472         l->header = header;
8473
8474         for (i = 0; i < params->n_actions; i++) {
8475                 const char *mf_name = params->action_field_names[i];
8476
8477                 l->actions[i] = action_find(p, params->action_names[i]);
8478
8479                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8480         }
8481
8482         l->default_action = default_action;
8483
8484         if (default_action->st)
8485                 memcpy(l->default_action_data,
8486                        params->default_action_data,
8487                        default_action->st->n_bits / 8);
8488
8489         l->n_actions = params->n_actions;
8490
8491         l->default_action_is_const = params->default_action_is_const;
8492
8493         l->action_data_size_max = action_data_size_max;
8494
8495         l->size = size;
8496
8497         l->timeout = timeout;
8498
8499         l->id = p->n_learners;
8500
8501         /* Node add to tailq. */
8502         TAILQ_INSERT_TAIL(&p->learners, l, node);
8503         p->n_learners++;
8504
8505         return 0;
8506
8507 nomem:
8508         if (!l)
8509                 return -ENOMEM;
8510
8511         free(l->action_arg);
8512         free(l->actions);
8513         free(l->fields);
8514         free(l);
8515
8516         return -ENOMEM;
8517 }
8518
8519 static void
8520 learner_params_free(struct rte_swx_table_learner_params *params)
8521 {
8522         if (!params)
8523                 return;
8524
8525         free(params->key_mask0);
8526
8527         free(params);
8528 }
8529
8530 static struct rte_swx_table_learner_params *
8531 learner_params_get(struct learner *l)
8532 {
8533         struct rte_swx_table_learner_params *params = NULL;
8534         struct field *first, *last;
8535         uint32_t i;
8536
8537         /* Memory allocation. */
8538         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8539         if (!params)
8540                 goto error;
8541
8542         /* Find first (smallest offset) and last (biggest offset) match fields. */
8543         first = l->fields[0];
8544         last = l->fields[0];
8545
8546         for (i = 0; i < l->n_fields; i++) {
8547                 struct field *f = l->fields[i];
8548
8549                 if (f->offset < first->offset)
8550                         first = f;
8551
8552                 if (f->offset > last->offset)
8553                         last = f;
8554         }
8555
8556         /* Key offset and size. */
8557         params->key_offset = first->offset / 8;
8558         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8559
8560         /* Memory allocation. */
8561         params->key_mask0 = calloc(1, params->key_size);
8562         if (!params->key_mask0)
8563                 goto error;
8564
8565         /* Key mask. */
8566         for (i = 0; i < l->n_fields; i++) {
8567                 struct field *f = l->fields[i];
8568                 uint32_t start = (f->offset - first->offset) / 8;
8569                 size_t size = f->n_bits / 8;
8570
8571                 memset(&params->key_mask0[start], 0xFF, size);
8572         }
8573
8574         /* Action data size. */
8575         params->action_data_size = l->action_data_size_max;
8576
8577         /* Maximum number of keys. */
8578         params->n_keys_max = l->size;
8579
8580         /* Timeout. */
8581         params->key_timeout = l->timeout;
8582
8583         return params;
8584
8585 error:
8586         learner_params_free(params);
8587         return NULL;
8588 }
8589
8590 static void
8591 learner_build_free(struct rte_swx_pipeline *p)
8592 {
8593         uint32_t i;
8594
8595         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8596                 struct thread *t = &p->threads[i];
8597                 uint32_t j;
8598
8599                 if (!t->learners)
8600                         continue;
8601
8602                 for (j = 0; j < p->n_learners; j++) {
8603                         struct learner_runtime *r = &t->learners[j];
8604
8605                         free(r->mailbox);
8606                         free(r->action_data);
8607                 }
8608
8609                 free(t->learners);
8610                 t->learners = NULL;
8611         }
8612
8613         if (p->learner_stats) {
8614                 for (i = 0; i < p->n_learners; i++)
8615                         free(p->learner_stats[i].n_pkts_action);
8616
8617                 free(p->learner_stats);
8618         }
8619 }
8620
8621 static int
8622 learner_build(struct rte_swx_pipeline *p)
8623 {
8624         uint32_t i;
8625         int status = 0;
8626
8627         /* Per pipeline: learner statistics. */
8628         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8629         CHECK(p->learner_stats, ENOMEM);
8630
8631         for (i = 0; i < p->n_learners; i++) {
8632                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8633                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8634         }
8635
8636         /* Per thread: learner run-time. */
8637         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8638                 struct thread *t = &p->threads[i];
8639                 struct learner *l;
8640
8641                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8642                 if (!t->learners) {
8643                         status = -ENOMEM;
8644                         goto error;
8645                 }
8646
8647                 TAILQ_FOREACH(l, &p->learners, node) {
8648                         struct learner_runtime *r = &t->learners[l->id];
8649                         uint64_t size;
8650                         uint32_t j;
8651
8652                         /* r->mailbox. */
8653                         size = rte_swx_table_learner_mailbox_size_get();
8654                         if (size) {
8655                                 r->mailbox = calloc(1, size);
8656                                 if (!r->mailbox) {
8657                                         status = -ENOMEM;
8658                                         goto error;
8659                                 }
8660                         }
8661
8662                         /* r->key. */
8663                         r->key = l->header ?
8664                                 &t->structs[l->header->struct_id] :
8665                                 &t->structs[p->metadata_struct_id];
8666
8667                         /* r->action_data. */
8668                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8669                         if (!r->action_data) {
8670                                 status = -ENOMEM;
8671                                 goto error;
8672                         }
8673
8674                         for (j = 0; j < l->n_actions; j++) {
8675                                 struct action *a = l->actions[j];
8676                                 struct field *mf = l->action_arg[j];
8677                                 uint8_t *m = t->structs[p->metadata_struct_id];
8678
8679                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8680                         }
8681                 }
8682         }
8683
8684         return 0;
8685
8686 error:
8687         learner_build_free(p);
8688         return status;
8689 }
8690
8691 static void
8692 learner_free(struct rte_swx_pipeline *p)
8693 {
8694         learner_build_free(p);
8695
8696         /* Learner tables. */
8697         for ( ; ; ) {
8698                 struct learner *l;
8699
8700                 l = TAILQ_FIRST(&p->learners);
8701                 if (!l)
8702                         break;
8703
8704                 TAILQ_REMOVE(&p->learners, l, node);
8705                 free(l->fields);
8706                 free(l->actions);
8707                 free(l->action_arg);
8708                 free(l->default_action_data);
8709                 free(l);
8710         }
8711 }
8712
8713 /*
8714  * Table state.
8715  */
8716 static int
8717 table_state_build(struct rte_swx_pipeline *p)
8718 {
8719         struct table *table;
8720         struct selector *s;
8721         struct learner *l;
8722
8723         p->table_state = calloc(p->n_tables + p->n_selectors,
8724                                 sizeof(struct rte_swx_table_state));
8725         CHECK(p->table_state, ENOMEM);
8726
8727         TAILQ_FOREACH(table, &p->tables, node) {
8728                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8729
8730                 if (table->type) {
8731                         struct rte_swx_table_params *params;
8732
8733                         /* ts->obj. */
8734                         params = table_params_get(table);
8735                         CHECK(params, ENOMEM);
8736
8737                         ts->obj = table->type->ops.create(params,
8738                                 NULL,
8739                                 table->args,
8740                                 p->numa_node);
8741
8742                         table_params_free(params);
8743                         CHECK(ts->obj, ENODEV);
8744                 }
8745
8746                 /* ts->default_action_data. */
8747                 if (table->action_data_size_max) {
8748                         ts->default_action_data =
8749                                 malloc(table->action_data_size_max);
8750                         CHECK(ts->default_action_data, ENOMEM);
8751
8752                         memcpy(ts->default_action_data,
8753                                table->default_action_data,
8754                                table->action_data_size_max);
8755                 }
8756
8757                 /* ts->default_action_id. */
8758                 ts->default_action_id = table->default_action->id;
8759         }
8760
8761         TAILQ_FOREACH(s, &p->selectors, node) {
8762                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8763                 struct rte_swx_table_selector_params *params;
8764
8765                 /* ts->obj. */
8766                 params = selector_table_params_get(s);
8767                 CHECK(params, ENOMEM);
8768
8769                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8770
8771                 selector_params_free(params);
8772                 CHECK(ts->obj, ENODEV);
8773         }
8774
8775         TAILQ_FOREACH(l, &p->learners, node) {
8776                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8777                         p->n_selectors + l->id];
8778                 struct rte_swx_table_learner_params *params;
8779
8780                 /* ts->obj. */
8781                 params = learner_params_get(l);
8782                 CHECK(params, ENOMEM);
8783
8784                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8785                 learner_params_free(params);
8786                 CHECK(ts->obj, ENODEV);
8787
8788                 /* ts->default_action_data. */
8789                 if (l->action_data_size_max) {
8790                         ts->default_action_data = malloc(l->action_data_size_max);
8791                         CHECK(ts->default_action_data, ENOMEM);
8792
8793                         memcpy(ts->default_action_data,
8794                                l->default_action_data,
8795                                l->action_data_size_max);
8796                 }
8797
8798                 /* ts->default_action_id. */
8799                 ts->default_action_id = l->default_action->id;
8800         }
8801
8802         return 0;
8803 }
8804
8805 static void
8806 table_state_build_free(struct rte_swx_pipeline *p)
8807 {
8808         uint32_t i;
8809
8810         if (!p->table_state)
8811                 return;
8812
8813         for (i = 0; i < p->n_tables; i++) {
8814                 struct rte_swx_table_state *ts = &p->table_state[i];
8815                 struct table *table = table_find_by_id(p, i);
8816
8817                 /* ts->obj. */
8818                 if (table->type && ts->obj)
8819                         table->type->ops.free(ts->obj);
8820
8821                 /* ts->default_action_data. */
8822                 free(ts->default_action_data);
8823         }
8824
8825         for (i = 0; i < p->n_selectors; i++) {
8826                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8827
8828                 /* ts->obj. */
8829                 if (ts->obj)
8830                         rte_swx_table_selector_free(ts->obj);
8831         }
8832
8833         for (i = 0; i < p->n_learners; i++) {
8834                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8835
8836                 /* ts->obj. */
8837                 if (ts->obj)
8838                         rte_swx_table_learner_free(ts->obj);
8839
8840                 /* ts->default_action_data. */
8841                 free(ts->default_action_data);
8842         }
8843
8844         free(p->table_state);
8845         p->table_state = NULL;
8846 }
8847
8848 static void
8849 table_state_free(struct rte_swx_pipeline *p)
8850 {
8851         table_state_build_free(p);
8852 }
8853
8854 /*
8855  * Register array.
8856  */
8857 static struct regarray *
8858 regarray_find(struct rte_swx_pipeline *p, const char *name)
8859 {
8860         struct regarray *elem;
8861
8862         TAILQ_FOREACH(elem, &p->regarrays, node)
8863                 if (!strcmp(elem->name, name))
8864                         return elem;
8865
8866         return NULL;
8867 }
8868
8869 static struct regarray *
8870 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8871 {
8872         struct regarray *elem = NULL;
8873
8874         TAILQ_FOREACH(elem, &p->regarrays, node)
8875                 if (elem->id == id)
8876                         return elem;
8877
8878         return NULL;
8879 }
8880
8881 int
8882 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8883                               const char *name,
8884                               uint32_t size,
8885                               uint64_t init_val)
8886 {
8887         struct regarray *r;
8888
8889         CHECK(p, EINVAL);
8890
8891         CHECK_NAME(name, EINVAL);
8892         CHECK(!regarray_find(p, name), EEXIST);
8893
8894         CHECK(size, EINVAL);
8895         size = rte_align32pow2(size);
8896
8897         /* Memory allocation. */
8898         r = calloc(1, sizeof(struct regarray));
8899         CHECK(r, ENOMEM);
8900
8901         /* Node initialization. */
8902         strcpy(r->name, name);
8903         r->init_val = init_val;
8904         r->size = size;
8905         r->id = p->n_regarrays;
8906
8907         /* Node add to tailq. */
8908         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8909         p->n_regarrays++;
8910
8911         return 0;
8912 }
8913
8914 static int
8915 regarray_build(struct rte_swx_pipeline *p)
8916 {
8917         struct regarray *regarray;
8918
8919         if (!p->n_regarrays)
8920                 return 0;
8921
8922         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8923         CHECK(p->regarray_runtime, ENOMEM);
8924
8925         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8926                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8927                 uint32_t i;
8928
8929                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8930                                          RTE_CACHE_LINE_SIZE,
8931                                          p->numa_node);
8932                 CHECK(r->regarray, ENOMEM);
8933
8934                 if (regarray->init_val)
8935                         for (i = 0; i < regarray->size; i++)
8936                                 r->regarray[i] = regarray->init_val;
8937
8938                 r->size_mask = regarray->size - 1;
8939         }
8940
8941         return 0;
8942 }
8943
8944 static void
8945 regarray_build_free(struct rte_swx_pipeline *p)
8946 {
8947         uint32_t i;
8948
8949         if (!p->regarray_runtime)
8950                 return;
8951
8952         for (i = 0; i < p->n_regarrays; i++) {
8953                 struct regarray *regarray = regarray_find_by_id(p, i);
8954                 struct regarray_runtime *r = &p->regarray_runtime[i];
8955
8956                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8957         }
8958
8959         free(p->regarray_runtime);
8960         p->regarray_runtime = NULL;
8961 }
8962
8963 static void
8964 regarray_free(struct rte_swx_pipeline *p)
8965 {
8966         regarray_build_free(p);
8967
8968         for ( ; ; ) {
8969                 struct regarray *elem;
8970
8971                 elem = TAILQ_FIRST(&p->regarrays);
8972                 if (!elem)
8973                         break;
8974
8975                 TAILQ_REMOVE(&p->regarrays, elem, node);
8976                 free(elem);
8977         }
8978 }
8979
8980 /*
8981  * Meter array.
8982  */
8983 static struct meter_profile *
8984 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8985 {
8986         struct meter_profile *elem;
8987
8988         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8989                 if (!strcmp(elem->name, name))
8990                         return elem;
8991
8992         return NULL;
8993 }
8994
8995 static struct metarray *
8996 metarray_find(struct rte_swx_pipeline *p, const char *name)
8997 {
8998         struct metarray *elem;
8999
9000         TAILQ_FOREACH(elem, &p->metarrays, node)
9001                 if (!strcmp(elem->name, name))
9002                         return elem;
9003
9004         return NULL;
9005 }
9006
9007 static struct metarray *
9008 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9009 {
9010         struct metarray *elem = NULL;
9011
9012         TAILQ_FOREACH(elem, &p->metarrays, node)
9013                 if (elem->id == id)
9014                         return elem;
9015
9016         return NULL;
9017 }
9018
9019 int
9020 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9021                                  const char *name,
9022                                  uint32_t size)
9023 {
9024         struct metarray *m;
9025
9026         CHECK(p, EINVAL);
9027
9028         CHECK_NAME(name, EINVAL);
9029         CHECK(!metarray_find(p, name), EEXIST);
9030
9031         CHECK(size, EINVAL);
9032         size = rte_align32pow2(size);
9033
9034         /* Memory allocation. */
9035         m = calloc(1, sizeof(struct metarray));
9036         CHECK(m, ENOMEM);
9037
9038         /* Node initialization. */
9039         strcpy(m->name, name);
9040         m->size = size;
9041         m->id = p->n_metarrays;
9042
9043         /* Node add to tailq. */
9044         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9045         p->n_metarrays++;
9046
9047         return 0;
9048 }
9049
9050 struct meter_profile meter_profile_default = {
9051         .node = {0},
9052         .name = "",
9053         .params = {0},
9054
9055         .profile = {
9056                 .cbs = 10000,
9057                 .pbs = 10000,
9058                 .cir_period = 1,
9059                 .cir_bytes_per_period = 1,
9060                 .pir_period = 1,
9061                 .pir_bytes_per_period = 1,
9062         },
9063
9064         .n_users = 0,
9065 };
9066
9067 static void
9068 meter_init(struct meter *m)
9069 {
9070         memset(m, 0, sizeof(struct meter));
9071         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9072         m->profile = &meter_profile_default;
9073         m->color_mask = RTE_COLOR_GREEN;
9074
9075         meter_profile_default.n_users++;
9076 }
9077
9078 static int
9079 metarray_build(struct rte_swx_pipeline *p)
9080 {
9081         struct metarray *m;
9082
9083         if (!p->n_metarrays)
9084                 return 0;
9085
9086         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9087         CHECK(p->metarray_runtime, ENOMEM);
9088
9089         TAILQ_FOREACH(m, &p->metarrays, node) {
9090                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9091                 uint32_t i;
9092
9093                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9094                                          RTE_CACHE_LINE_SIZE,
9095                                          p->numa_node);
9096                 CHECK(r->metarray, ENOMEM);
9097
9098                 for (i = 0; i < m->size; i++)
9099                         meter_init(&r->metarray[i]);
9100
9101                 r->size_mask = m->size - 1;
9102         }
9103
9104         return 0;
9105 }
9106
9107 static void
9108 metarray_build_free(struct rte_swx_pipeline *p)
9109 {
9110         uint32_t i;
9111
9112         if (!p->metarray_runtime)
9113                 return;
9114
9115         for (i = 0; i < p->n_metarrays; i++) {
9116                 struct metarray *m = metarray_find_by_id(p, i);
9117                 struct metarray_runtime *r = &p->metarray_runtime[i];
9118
9119                 env_free(r->metarray, m->size * sizeof(struct meter));
9120         }
9121
9122         free(p->metarray_runtime);
9123         p->metarray_runtime = NULL;
9124 }
9125
9126 static void
9127 metarray_free(struct rte_swx_pipeline *p)
9128 {
9129         metarray_build_free(p);
9130
9131         /* Meter arrays. */
9132         for ( ; ; ) {
9133                 struct metarray *elem;
9134
9135                 elem = TAILQ_FIRST(&p->metarrays);
9136                 if (!elem)
9137                         break;
9138
9139                 TAILQ_REMOVE(&p->metarrays, elem, node);
9140                 free(elem);
9141         }
9142
9143         /* Meter profiles. */
9144         for ( ; ; ) {
9145                 struct meter_profile *elem;
9146
9147                 elem = TAILQ_FIRST(&p->meter_profiles);
9148                 if (!elem)
9149                         break;
9150
9151                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9152                 free(elem);
9153         }
9154 }
9155
9156 /*
9157  * Pipeline.
9158  */
9159 int
9160 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9161 {
9162         struct rte_swx_pipeline *pipeline;
9163
9164         /* Check input parameters. */
9165         CHECK(p, EINVAL);
9166
9167         /* Memory allocation. */
9168         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9169         CHECK(pipeline, ENOMEM);
9170
9171         /* Initialization. */
9172         TAILQ_INIT(&pipeline->struct_types);
9173         TAILQ_INIT(&pipeline->port_in_types);
9174         TAILQ_INIT(&pipeline->ports_in);
9175         TAILQ_INIT(&pipeline->port_out_types);
9176         TAILQ_INIT(&pipeline->ports_out);
9177         TAILQ_INIT(&pipeline->extern_types);
9178         TAILQ_INIT(&pipeline->extern_objs);
9179         TAILQ_INIT(&pipeline->extern_funcs);
9180         TAILQ_INIT(&pipeline->headers);
9181         TAILQ_INIT(&pipeline->actions);
9182         TAILQ_INIT(&pipeline->table_types);
9183         TAILQ_INIT(&pipeline->tables);
9184         TAILQ_INIT(&pipeline->selectors);
9185         TAILQ_INIT(&pipeline->learners);
9186         TAILQ_INIT(&pipeline->regarrays);
9187         TAILQ_INIT(&pipeline->meter_profiles);
9188         TAILQ_INIT(&pipeline->metarrays);
9189
9190         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9191         pipeline->numa_node = numa_node;
9192
9193         *p = pipeline;
9194         return 0;
9195 }
9196
9197 void
9198 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9199 {
9200         if (!p)
9201                 return;
9202
9203         free(p->instructions);
9204
9205         metarray_free(p);
9206         regarray_free(p);
9207         table_state_free(p);
9208         learner_free(p);
9209         selector_free(p);
9210         table_free(p);
9211         action_free(p);
9212         metadata_free(p);
9213         header_free(p);
9214         extern_func_free(p);
9215         extern_obj_free(p);
9216         port_out_free(p);
9217         port_in_free(p);
9218         struct_free(p);
9219
9220         free(p);
9221 }
9222
9223 int
9224 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9225                                      const char **instructions,
9226                                      uint32_t n_instructions)
9227 {
9228         int err;
9229         uint32_t i;
9230
9231         err = instruction_config(p, NULL, instructions, n_instructions);
9232         if (err)
9233                 return err;
9234
9235         /* Thread instruction pointer reset. */
9236         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9237                 struct thread *t = &p->threads[i];
9238
9239                 thread_ip_reset(p, t);
9240         }
9241
9242         return 0;
9243 }
9244
9245 int
9246 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9247 {
9248         int status;
9249
9250         CHECK(p, EINVAL);
9251         CHECK(p->build_done == 0, EEXIST);
9252
9253         status = port_in_build(p);
9254         if (status)
9255                 goto error;
9256
9257         status = port_out_build(p);
9258         if (status)
9259                 goto error;
9260
9261         status = struct_build(p);
9262         if (status)
9263                 goto error;
9264
9265         status = extern_obj_build(p);
9266         if (status)
9267                 goto error;
9268
9269         status = extern_func_build(p);
9270         if (status)
9271                 goto error;
9272
9273         status = header_build(p);
9274         if (status)
9275                 goto error;
9276
9277         status = metadata_build(p);
9278         if (status)
9279                 goto error;
9280
9281         status = action_build(p);
9282         if (status)
9283                 goto error;
9284
9285         status = table_build(p);
9286         if (status)
9287                 goto error;
9288
9289         status = selector_build(p);
9290         if (status)
9291                 goto error;
9292
9293         status = learner_build(p);
9294         if (status)
9295                 goto error;
9296
9297         status = table_state_build(p);
9298         if (status)
9299                 goto error;
9300
9301         status = regarray_build(p);
9302         if (status)
9303                 goto error;
9304
9305         status = metarray_build(p);
9306         if (status)
9307                 goto error;
9308
9309         p->build_done = 1;
9310         return 0;
9311
9312 error:
9313         metarray_build_free(p);
9314         regarray_build_free(p);
9315         table_state_build_free(p);
9316         learner_build_free(p);
9317         selector_build_free(p);
9318         table_build_free(p);
9319         action_build_free(p);
9320         metadata_build_free(p);
9321         header_build_free(p);
9322         extern_func_build_free(p);
9323         extern_obj_build_free(p);
9324         port_out_build_free(p);
9325         port_in_build_free(p);
9326         struct_build_free(p);
9327
9328         return status;
9329 }
9330
9331 void
9332 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9333 {
9334         uint32_t i;
9335
9336         for (i = 0; i < n_instructions; i++)
9337                 instr_exec(p);
9338 }
9339
9340 void
9341 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9342 {
9343         uint32_t i;
9344
9345         for (i = 0; i < p->n_ports_out; i++) {
9346                 struct port_out_runtime *port = &p->out[i];
9347
9348                 if (port->flush)
9349                         port->flush(port->obj);
9350         }
9351 }
9352
9353 /*
9354  * Control.
9355  */
9356 int
9357 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9358                               struct rte_swx_ctl_pipeline_info *pipeline)
9359 {
9360         struct action *action;
9361         struct table *table;
9362         uint32_t n_actions = 0, n_tables = 0;
9363
9364         if (!p || !pipeline)
9365                 return -EINVAL;
9366
9367         TAILQ_FOREACH(action, &p->actions, node)
9368                 n_actions++;
9369
9370         TAILQ_FOREACH(table, &p->tables, node)
9371                 n_tables++;
9372
9373         pipeline->n_ports_in = p->n_ports_in;
9374         pipeline->n_ports_out = p->n_ports_out;
9375         pipeline->n_actions = n_actions;
9376         pipeline->n_tables = n_tables;
9377         pipeline->n_selectors = p->n_selectors;
9378         pipeline->n_learners = p->n_learners;
9379         pipeline->n_regarrays = p->n_regarrays;
9380         pipeline->n_metarrays = p->n_metarrays;
9381
9382         return 0;
9383 }
9384
9385 int
9386 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9387 {
9388         if (!p || !numa_node)
9389                 return -EINVAL;
9390
9391         *numa_node = p->numa_node;
9392         return 0;
9393 }
9394
9395 int
9396 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9397                             uint32_t action_id,
9398                             struct rte_swx_ctl_action_info *action)
9399 {
9400         struct action *a = NULL;
9401
9402         if (!p || (action_id >= p->n_actions) || !action)
9403                 return -EINVAL;
9404
9405         a = action_find_by_id(p, action_id);
9406         if (!a)
9407                 return -EINVAL;
9408
9409         strcpy(action->name, a->name);
9410         action->n_args = a->st ? a->st->n_fields : 0;
9411         return 0;
9412 }
9413
9414 int
9415 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9416                                 uint32_t action_id,
9417                                 uint32_t action_arg_id,
9418                                 struct rte_swx_ctl_action_arg_info *action_arg)
9419 {
9420         struct action *a = NULL;
9421         struct field *arg = NULL;
9422
9423         if (!p || (action_id >= p->n_actions) || !action_arg)
9424                 return -EINVAL;
9425
9426         a = action_find_by_id(p, action_id);
9427         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9428                 return -EINVAL;
9429
9430         arg = &a->st->fields[action_arg_id];
9431         strcpy(action_arg->name, arg->name);
9432         action_arg->n_bits = arg->n_bits;
9433         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9434
9435         return 0;
9436 }
9437
9438 int
9439 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9440                            uint32_t table_id,
9441                            struct rte_swx_ctl_table_info *table)
9442 {
9443         struct table *t = NULL;
9444
9445         if (!p || !table)
9446                 return -EINVAL;
9447
9448         t = table_find_by_id(p, table_id);
9449         if (!t)
9450                 return -EINVAL;
9451
9452         strcpy(table->name, t->name);
9453         strcpy(table->args, t->args);
9454         table->n_match_fields = t->n_fields;
9455         table->n_actions = t->n_actions;
9456         table->default_action_is_const = t->default_action_is_const;
9457         table->size = t->size;
9458         return 0;
9459 }
9460
9461 int
9462 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9463         uint32_t table_id,
9464         uint32_t match_field_id,
9465         struct rte_swx_ctl_table_match_field_info *match_field)
9466 {
9467         struct table *t;
9468         struct match_field *f;
9469
9470         if (!p || (table_id >= p->n_tables) || !match_field)
9471                 return -EINVAL;
9472
9473         t = table_find_by_id(p, table_id);
9474         if (!t || (match_field_id >= t->n_fields))
9475                 return -EINVAL;
9476
9477         f = &t->fields[match_field_id];
9478         match_field->match_type = f->match_type;
9479         match_field->is_header = t->header ? 1 : 0;
9480         match_field->n_bits = f->field->n_bits;
9481         match_field->offset = f->field->offset;
9482
9483         return 0;
9484 }
9485
9486 int
9487 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9488         uint32_t table_id,
9489         uint32_t table_action_id,
9490         struct rte_swx_ctl_table_action_info *table_action)
9491 {
9492         struct table *t;
9493
9494         if (!p || (table_id >= p->n_tables) || !table_action)
9495                 return -EINVAL;
9496
9497         t = table_find_by_id(p, table_id);
9498         if (!t || (table_action_id >= t->n_actions))
9499                 return -EINVAL;
9500
9501         table_action->action_id = t->actions[table_action_id]->id;
9502
9503         return 0;
9504 }
9505
9506 int
9507 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9508                           uint32_t table_id,
9509                           struct rte_swx_table_ops *table_ops,
9510                           int *is_stub)
9511 {
9512         struct table *t;
9513
9514         if (!p || (table_id >= p->n_tables))
9515                 return -EINVAL;
9516
9517         t = table_find_by_id(p, table_id);
9518         if (!t)
9519                 return -EINVAL;
9520
9521         if (t->type) {
9522                 if (table_ops)
9523                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9524                 *is_stub = 0;
9525         } else {
9526                 *is_stub = 1;
9527         }
9528
9529         return 0;
9530 }
9531
9532 int
9533 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9534                               uint32_t selector_id,
9535                               struct rte_swx_ctl_selector_info *selector)
9536 {
9537         struct selector *s = NULL;
9538
9539         if (!p || !selector)
9540                 return -EINVAL;
9541
9542         s = selector_find_by_id(p, selector_id);
9543         if (!s)
9544                 return -EINVAL;
9545
9546         strcpy(selector->name, s->name);
9547
9548         selector->n_selector_fields = s->n_selector_fields;
9549         selector->n_groups_max = s->n_groups_max;
9550         selector->n_members_per_group_max = s->n_members_per_group_max;
9551
9552         return 0;
9553 }
9554
9555 int
9556 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9557          uint32_t selector_id,
9558          struct rte_swx_ctl_table_match_field_info *field)
9559 {
9560         struct selector *s;
9561
9562         if (!p || (selector_id >= p->n_selectors) || !field)
9563                 return -EINVAL;
9564
9565         s = selector_find_by_id(p, selector_id);
9566         if (!s)
9567                 return -EINVAL;
9568
9569         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9570         field->is_header = 0;
9571         field->n_bits = s->group_id_field->n_bits;
9572         field->offset = s->group_id_field->offset;
9573
9574         return 0;
9575 }
9576
9577 int
9578 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9579          uint32_t selector_id,
9580          uint32_t selector_field_id,
9581          struct rte_swx_ctl_table_match_field_info *field)
9582 {
9583         struct selector *s;
9584         struct field *f;
9585
9586         if (!p || (selector_id >= p->n_selectors) || !field)
9587                 return -EINVAL;
9588
9589         s = selector_find_by_id(p, selector_id);
9590         if (!s || (selector_field_id >= s->n_selector_fields))
9591                 return -EINVAL;
9592
9593         f = s->selector_fields[selector_field_id];
9594         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9595         field->is_header = s->selector_header ? 1 : 0;
9596         field->n_bits = f->n_bits;
9597         field->offset = f->offset;
9598
9599         return 0;
9600 }
9601
9602 int
9603 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9604          uint32_t selector_id,
9605          struct rte_swx_ctl_table_match_field_info *field)
9606 {
9607         struct selector *s;
9608
9609         if (!p || (selector_id >= p->n_selectors) || !field)
9610                 return -EINVAL;
9611
9612         s = selector_find_by_id(p, selector_id);
9613         if (!s)
9614                 return -EINVAL;
9615
9616         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9617         field->is_header = 0;
9618         field->n_bits = s->member_id_field->n_bits;
9619         field->offset = s->member_id_field->offset;
9620
9621         return 0;
9622 }
9623
9624 int
9625 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9626                              uint32_t learner_id,
9627                              struct rte_swx_ctl_learner_info *learner)
9628 {
9629         struct learner *l = NULL;
9630
9631         if (!p || !learner)
9632                 return -EINVAL;
9633
9634         l = learner_find_by_id(p, learner_id);
9635         if (!l)
9636                 return -EINVAL;
9637
9638         strcpy(learner->name, l->name);
9639
9640         learner->n_match_fields = l->n_fields;
9641         learner->n_actions = l->n_actions;
9642         learner->default_action_is_const = l->default_action_is_const;
9643         learner->size = l->size;
9644
9645         return 0;
9646 }
9647
9648 int
9649 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9650                                          uint32_t learner_id,
9651                                          uint32_t match_field_id,
9652                                          struct rte_swx_ctl_table_match_field_info *match_field)
9653 {
9654         struct learner *l;
9655         struct field *f;
9656
9657         if (!p || (learner_id >= p->n_learners) || !match_field)
9658                 return -EINVAL;
9659
9660         l = learner_find_by_id(p, learner_id);
9661         if (!l || (match_field_id >= l->n_fields))
9662                 return -EINVAL;
9663
9664         f = l->fields[match_field_id];
9665         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9666         match_field->is_header = l->header ? 1 : 0;
9667         match_field->n_bits = f->n_bits;
9668         match_field->offset = f->offset;
9669
9670         return 0;
9671 }
9672
9673 int
9674 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9675                                     uint32_t learner_id,
9676                                     uint32_t learner_action_id,
9677                                     struct rte_swx_ctl_table_action_info *learner_action)
9678 {
9679         struct learner *l;
9680
9681         if (!p || (learner_id >= p->n_learners) || !learner_action)
9682                 return -EINVAL;
9683
9684         l = learner_find_by_id(p, learner_id);
9685         if (!l || (learner_action_id >= l->n_actions))
9686                 return -EINVAL;
9687
9688         learner_action->action_id = l->actions[learner_action_id]->id;
9689
9690         return 0;
9691 }
9692
9693 int
9694 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9695                                  struct rte_swx_table_state **table_state)
9696 {
9697         if (!p || !table_state || !p->build_done)
9698                 return -EINVAL;
9699
9700         *table_state = p->table_state;
9701         return 0;
9702 }
9703
9704 int
9705 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9706                                  struct rte_swx_table_state *table_state)
9707 {
9708         if (!p || !table_state || !p->build_done)
9709                 return -EINVAL;
9710
9711         p->table_state = table_state;
9712         return 0;
9713 }
9714
9715 int
9716 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9717                                         uint32_t port_id,
9718                                         struct rte_swx_port_in_stats *stats)
9719 {
9720         struct port_in *port;
9721
9722         if (!p || !stats)
9723                 return -EINVAL;
9724
9725         port = port_in_find(p, port_id);
9726         if (!port)
9727                 return -EINVAL;
9728
9729         port->type->ops.stats_read(port->obj, stats);
9730         return 0;
9731 }
9732
9733 int
9734 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9735                                          uint32_t port_id,
9736                                          struct rte_swx_port_out_stats *stats)
9737 {
9738         struct port_out *port;
9739
9740         if (!p || !stats)
9741                 return -EINVAL;
9742
9743         port = port_out_find(p, port_id);
9744         if (!port)
9745                 return -EINVAL;
9746
9747         port->type->ops.stats_read(port->obj, stats);
9748         return 0;
9749 }
9750
9751 int
9752 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9753                                       const char *table_name,
9754                                       struct rte_swx_table_stats *stats)
9755 {
9756         struct table *table;
9757         struct table_statistics *table_stats;
9758
9759         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9760                 return -EINVAL;
9761
9762         table = table_find(p, table_name);
9763         if (!table)
9764                 return -EINVAL;
9765
9766         table_stats = &p->table_stats[table->id];
9767
9768         memcpy(stats->n_pkts_action,
9769                table_stats->n_pkts_action,
9770                p->n_actions * sizeof(uint64_t));
9771
9772         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9773         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9774
9775         return 0;
9776 }
9777
9778 int
9779 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9780         const char *selector_name,
9781         struct rte_swx_pipeline_selector_stats *stats)
9782 {
9783         struct selector *s;
9784
9785         if (!p || !selector_name || !selector_name[0] || !stats)
9786                 return -EINVAL;
9787
9788         s = selector_find(p, selector_name);
9789         if (!s)
9790                 return -EINVAL;
9791
9792         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9793
9794         return 0;
9795 }
9796
9797 int
9798 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9799                                         const char *learner_name,
9800                                         struct rte_swx_learner_stats *stats)
9801 {
9802         struct learner *l;
9803         struct learner_statistics *learner_stats;
9804
9805         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9806                 return -EINVAL;
9807
9808         l = learner_find(p, learner_name);
9809         if (!l)
9810                 return -EINVAL;
9811
9812         learner_stats = &p->learner_stats[l->id];
9813
9814         memcpy(stats->n_pkts_action,
9815                learner_stats->n_pkts_action,
9816                p->n_actions * sizeof(uint64_t));
9817
9818         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9819         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9820
9821         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9822         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9823
9824         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9825
9826         return 0;
9827 }
9828
9829 int
9830 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9831                               uint32_t regarray_id,
9832                               struct rte_swx_ctl_regarray_info *regarray)
9833 {
9834         struct regarray *r;
9835
9836         if (!p || !regarray)
9837                 return -EINVAL;
9838
9839         r = regarray_find_by_id(p, regarray_id);
9840         if (!r)
9841                 return -EINVAL;
9842
9843         strcpy(regarray->name, r->name);
9844         regarray->size = r->size;
9845         return 0;
9846 }
9847
9848 int
9849 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9850                                    const char *regarray_name,
9851                                    uint32_t regarray_index,
9852                                    uint64_t *value)
9853 {
9854         struct regarray *regarray;
9855         struct regarray_runtime *r;
9856
9857         if (!p || !regarray_name || !value)
9858                 return -EINVAL;
9859
9860         regarray = regarray_find(p, regarray_name);
9861         if (!regarray || (regarray_index >= regarray->size))
9862                 return -EINVAL;
9863
9864         r = &p->regarray_runtime[regarray->id];
9865         *value = r->regarray[regarray_index];
9866         return 0;
9867 }
9868
9869 int
9870 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9871                                    const char *regarray_name,
9872                                    uint32_t regarray_index,
9873                                    uint64_t value)
9874 {
9875         struct regarray *regarray;
9876         struct regarray_runtime *r;
9877
9878         if (!p || !regarray_name)
9879                 return -EINVAL;
9880
9881         regarray = regarray_find(p, regarray_name);
9882         if (!regarray || (regarray_index >= regarray->size))
9883                 return -EINVAL;
9884
9885         r = &p->regarray_runtime[regarray->id];
9886         r->regarray[regarray_index] = value;
9887         return 0;
9888 }
9889
9890 int
9891 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9892                               uint32_t metarray_id,
9893                               struct rte_swx_ctl_metarray_info *metarray)
9894 {
9895         struct metarray *m;
9896
9897         if (!p || !metarray)
9898                 return -EINVAL;
9899
9900         m = metarray_find_by_id(p, metarray_id);
9901         if (!m)
9902                 return -EINVAL;
9903
9904         strcpy(metarray->name, m->name);
9905         metarray->size = m->size;
9906         return 0;
9907 }
9908
9909 int
9910 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9911                               const char *name,
9912                               struct rte_meter_trtcm_params *params)
9913 {
9914         struct meter_profile *mp;
9915         int status;
9916
9917         CHECK(p, EINVAL);
9918         CHECK_NAME(name, EINVAL);
9919         CHECK(params, EINVAL);
9920         CHECK(!meter_profile_find(p, name), EEXIST);
9921
9922         /* Node allocation. */
9923         mp = calloc(1, sizeof(struct meter_profile));
9924         CHECK(mp, ENOMEM);
9925
9926         /* Node initialization. */
9927         strcpy(mp->name, name);
9928         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9929         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9930         if (status) {
9931                 free(mp);
9932                 CHECK(0, EINVAL);
9933         }
9934
9935         /* Node add to tailq. */
9936         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9937
9938         return 0;
9939 }
9940
9941 int
9942 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9943                                  const char *name)
9944 {
9945         struct meter_profile *mp;
9946
9947         CHECK(p, EINVAL);
9948         CHECK_NAME(name, EINVAL);
9949
9950         mp = meter_profile_find(p, name);
9951         CHECK(mp, EINVAL);
9952         CHECK(!mp->n_users, EBUSY);
9953
9954         /* Remove node from tailq. */
9955         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9956         free(mp);
9957
9958         return 0;
9959 }
9960
9961 int
9962 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9963                         const char *metarray_name,
9964                         uint32_t metarray_index)
9965 {
9966         struct meter_profile *mp_old;
9967         struct metarray *metarray;
9968         struct metarray_runtime *metarray_runtime;
9969         struct meter *m;
9970
9971         CHECK(p, EINVAL);
9972         CHECK_NAME(metarray_name, EINVAL);
9973
9974         metarray = metarray_find(p, metarray_name);
9975         CHECK(metarray, EINVAL);
9976         CHECK(metarray_index < metarray->size, EINVAL);
9977
9978         metarray_runtime = &p->metarray_runtime[metarray->id];
9979         m = &metarray_runtime->metarray[metarray_index];
9980         mp_old = m->profile;
9981
9982         meter_init(m);
9983
9984         mp_old->n_users--;
9985
9986         return 0;
9987 }
9988
9989 int
9990 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9991                       const char *metarray_name,
9992                       uint32_t metarray_index,
9993                       const char *profile_name)
9994 {
9995         struct meter_profile *mp, *mp_old;
9996         struct metarray *metarray;
9997         struct metarray_runtime *metarray_runtime;
9998         struct meter *m;
9999
10000         CHECK(p, EINVAL);
10001         CHECK_NAME(metarray_name, EINVAL);
10002
10003         metarray = metarray_find(p, metarray_name);
10004         CHECK(metarray, EINVAL);
10005         CHECK(metarray_index < metarray->size, EINVAL);
10006
10007         mp = meter_profile_find(p, profile_name);
10008         CHECK(mp, EINVAL);
10009
10010         metarray_runtime = &p->metarray_runtime[metarray->id];
10011         m = &metarray_runtime->metarray[metarray_index];
10012         mp_old = m->profile;
10013
10014         memset(m, 0, sizeof(struct meter));
10015         rte_meter_trtcm_config(&m->m, &mp->profile);
10016         m->profile = mp;
10017         m->color_mask = RTE_COLORS;
10018
10019         mp->n_users++;
10020         mp_old->n_users--;
10021
10022         return 0;
10023 }
10024
10025 int
10026 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10027                              const char *metarray_name,
10028                              uint32_t metarray_index,
10029                              struct rte_swx_ctl_meter_stats *stats)
10030 {
10031         struct metarray *metarray;
10032         struct metarray_runtime *metarray_runtime;
10033         struct meter *m;
10034
10035         CHECK(p, EINVAL);
10036         CHECK_NAME(metarray_name, EINVAL);
10037
10038         metarray = metarray_find(p, metarray_name);
10039         CHECK(metarray, EINVAL);
10040         CHECK(metarray_index < metarray->size, EINVAL);
10041
10042         CHECK(stats, EINVAL);
10043
10044         metarray_runtime = &p->metarray_runtime[metarray->id];
10045         m = &metarray_runtime->metarray[metarray_index];
10046
10047         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10048         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10049
10050         return 0;
10051 }