lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
1669
1670 static inline void
1671 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
1672 {
1673         struct thread *t = &p->threads[p->thread_id];
1674         struct instruction *ip = t->ip;
1675         uint64_t valid_headers = t->valid_headers;
1676         uint8_t *ptr = t->ptr;
1677         uint32_t offset = t->pkt.offset;
1678         uint32_t length = t->pkt.length;
1679         uint32_t i;
1680
1681         for (i = 0; i < n_extract; i++) {
1682                 uint32_t header_id = ip->io.hdr.header_id[i];
1683                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1684                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1685
1686                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1687                       p->thread_id,
1688                       header_id,
1689                       n_bytes);
1690
1691                 /* Headers. */
1692                 t->structs[struct_id] = ptr;
1693                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1694
1695                 /* Packet. */
1696                 offset += n_bytes;
1697                 length -= n_bytes;
1698                 ptr += n_bytes;
1699         }
1700
1701         /* Headers. */
1702         t->valid_headers = valid_headers;
1703
1704         /* Packet. */
1705         t->pkt.offset = offset;
1706         t->pkt.length = length;
1707         t->ptr = ptr;
1708 }
1709
1710 static inline void
1711 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1712 {
1713         __instr_hdr_extract_exec(p, 1);
1714
1715         /* Thread. */
1716         thread_ip_inc(p);
1717 }
1718
1719 static inline void
1720 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1721 {
1722         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
1723               p->thread_id);
1724
1725         __instr_hdr_extract_exec(p, 2);
1726
1727         /* Thread. */
1728         thread_ip_inc(p);
1729 }
1730
1731 static inline void
1732 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1733 {
1734         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
1735               p->thread_id);
1736
1737         __instr_hdr_extract_exec(p, 3);
1738
1739         /* Thread. */
1740         thread_ip_inc(p);
1741 }
1742
1743 static inline void
1744 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1745 {
1746         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
1747               p->thread_id);
1748
1749         __instr_hdr_extract_exec(p, 4);
1750
1751         /* Thread. */
1752         thread_ip_inc(p);
1753 }
1754
1755 static inline void
1756 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1757 {
1758         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
1759               p->thread_id);
1760
1761         __instr_hdr_extract_exec(p, 5);
1762
1763         /* Thread. */
1764         thread_ip_inc(p);
1765 }
1766
1767 static inline void
1768 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1769 {
1770         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
1771               p->thread_id);
1772
1773         __instr_hdr_extract_exec(p, 6);
1774
1775         /* Thread. */
1776         thread_ip_inc(p);
1777 }
1778
1779 static inline void
1780 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1781 {
1782         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
1783               p->thread_id);
1784
1785         __instr_hdr_extract_exec(p, 7);
1786
1787         /* Thread. */
1788         thread_ip_inc(p);
1789 }
1790
1791 static inline void
1792 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1793 {
1794         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
1795               p->thread_id);
1796
1797         __instr_hdr_extract_exec(p, 8);
1798
1799         /* Thread. */
1800         thread_ip_inc(p);
1801 }
1802
1803 static inline void
1804 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1805 {
1806         struct thread *t = &p->threads[p->thread_id];
1807         struct instruction *ip = t->ip;
1808
1809         uint64_t valid_headers = t->valid_headers;
1810         uint8_t *ptr = t->ptr;
1811         uint32_t offset = t->pkt.offset;
1812         uint32_t length = t->pkt.length;
1813
1814         uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1815         uint32_t header_id = ip->io.hdr.header_id[0];
1816         uint32_t struct_id = ip->io.hdr.struct_id[0];
1817         uint32_t n_bytes = ip->io.hdr.n_bytes[0];
1818
1819         struct header_runtime *h = &t->headers[header_id];
1820
1821         TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
1822               p->thread_id,
1823               header_id,
1824               n_bytes,
1825               n_bytes_last);
1826
1827         n_bytes += n_bytes_last;
1828
1829         /* Headers. */
1830         t->structs[struct_id] = ptr;
1831         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1832         h->n_bytes = n_bytes;
1833
1834         /* Packet. */
1835         t->pkt.offset = offset + n_bytes;
1836         t->pkt.length = length - n_bytes;
1837         t->ptr = ptr + n_bytes;
1838
1839         /* Thread. */
1840         thread_ip_inc(p);
1841 }
1842
1843 static inline void
1844 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1845 {
1846         struct thread *t = &p->threads[p->thread_id];
1847         struct instruction *ip = t->ip;
1848
1849         uint64_t valid_headers = t->valid_headers;
1850         uint8_t *ptr = t->ptr;
1851
1852         uint32_t header_id = ip->io.hdr.header_id[0];
1853         uint32_t struct_id = ip->io.hdr.struct_id[0];
1854
1855         TRACE("[Thread %2u]: lookahead header %u\n",
1856               p->thread_id,
1857               header_id);
1858
1859         /* Headers. */
1860         t->structs[struct_id] = ptr;
1861         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1862
1863         /* Thread. */
1864         thread_ip_inc(p);
1865 }
1866
1867 /*
1868  * emit.
1869  */
1870 static int
1871 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1872                          struct action *action __rte_unused,
1873                          char **tokens,
1874                          int n_tokens,
1875                          struct instruction *instr,
1876                          struct instruction_data *data __rte_unused)
1877 {
1878         struct header *h;
1879
1880         CHECK(n_tokens == 2, EINVAL);
1881
1882         h = header_parse(p, tokens[1]);
1883         CHECK(h, EINVAL);
1884
1885         instr->type = INSTR_HDR_EMIT;
1886         instr->io.hdr.header_id[0] = h->id;
1887         instr->io.hdr.struct_id[0] = h->struct_id;
1888         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1889         return 0;
1890 }
1891
1892 static inline void
1893 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
1894
1895 static inline void
1896 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
1897 {
1898         struct thread *t = &p->threads[p->thread_id];
1899         struct instruction *ip = t->ip;
1900         uint64_t valid_headers = t->valid_headers;
1901         uint32_t n_headers_out = t->n_headers_out;
1902         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1903         uint8_t *ho_ptr = NULL;
1904         uint32_t ho_nbytes = 0, first = 1, i;
1905
1906         for (i = 0; i < n_emit; i++) {
1907                 uint32_t header_id = ip->io.hdr.header_id[i];
1908                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1909
1910                 struct header_runtime *hi = &t->headers[header_id];
1911                 uint8_t *hi_ptr0 = hi->ptr0;
1912                 uint32_t n_bytes = hi->n_bytes;
1913
1914                 uint8_t *hi_ptr = t->structs[struct_id];
1915
1916                 if (!MASK64_BIT_GET(valid_headers, header_id))
1917                         continue;
1918
1919                 TRACE("[Thread %2u]: emit header %u\n",
1920                       p->thread_id,
1921                       header_id);
1922
1923                 /* Headers. */
1924                 if (first) {
1925                         first = 0;
1926
1927                         if (!t->n_headers_out) {
1928                                 ho = &t->headers_out[0];
1929
1930                                 ho->ptr0 = hi_ptr0;
1931                                 ho->ptr = hi_ptr;
1932
1933                                 ho_ptr = hi_ptr;
1934                                 ho_nbytes = n_bytes;
1935
1936                                 n_headers_out = 1;
1937
1938                                 continue;
1939                         } else {
1940                                 ho_ptr = ho->ptr;
1941                                 ho_nbytes = ho->n_bytes;
1942                         }
1943                 }
1944
1945                 if (ho_ptr + ho_nbytes == hi_ptr) {
1946                         ho_nbytes += n_bytes;
1947                 } else {
1948                         ho->n_bytes = ho_nbytes;
1949
1950                         ho++;
1951                         ho->ptr0 = hi_ptr0;
1952                         ho->ptr = hi_ptr;
1953
1954                         ho_ptr = hi_ptr;
1955                         ho_nbytes = n_bytes;
1956
1957                         n_headers_out++;
1958                 }
1959         }
1960
1961         ho->n_bytes = ho_nbytes;
1962         t->n_headers_out = n_headers_out;
1963 }
1964
1965 static inline void
1966 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1967 {
1968         __instr_hdr_emit_exec(p, 1);
1969
1970         /* Thread. */
1971         thread_ip_inc(p);
1972 }
1973
1974 static inline void
1975 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1976 {
1977         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
1978               p->thread_id);
1979
1980         __instr_hdr_emit_exec(p, 1);
1981         instr_tx_exec(p);
1982 }
1983
1984 static inline void
1985 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1986 {
1987         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
1988               p->thread_id);
1989
1990         __instr_hdr_emit_exec(p, 2);
1991         instr_tx_exec(p);
1992 }
1993
1994 static inline void
1995 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1996 {
1997         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
1998               p->thread_id);
1999
2000         __instr_hdr_emit_exec(p, 3);
2001         instr_tx_exec(p);
2002 }
2003
2004 static inline void
2005 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2006 {
2007         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2008               p->thread_id);
2009
2010         __instr_hdr_emit_exec(p, 4);
2011         instr_tx_exec(p);
2012 }
2013
2014 static inline void
2015 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2016 {
2017         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2018               p->thread_id);
2019
2020         __instr_hdr_emit_exec(p, 5);
2021         instr_tx_exec(p);
2022 }
2023
2024 static inline void
2025 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2026 {
2027         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2028               p->thread_id);
2029
2030         __instr_hdr_emit_exec(p, 6);
2031         instr_tx_exec(p);
2032 }
2033
2034 static inline void
2035 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2036 {
2037         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2038               p->thread_id);
2039
2040         __instr_hdr_emit_exec(p, 7);
2041         instr_tx_exec(p);
2042 }
2043
2044 static inline void
2045 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2046 {
2047         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2048               p->thread_id);
2049
2050         __instr_hdr_emit_exec(p, 8);
2051         instr_tx_exec(p);
2052 }
2053
2054 /*
2055  * validate.
2056  */
2057 static int
2058 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2059                              struct action *action __rte_unused,
2060                              char **tokens,
2061                              int n_tokens,
2062                              struct instruction *instr,
2063                              struct instruction_data *data __rte_unused)
2064 {
2065         struct header *h;
2066
2067         CHECK(n_tokens == 2, EINVAL);
2068
2069         h = header_parse(p, tokens[1]);
2070         CHECK(h, EINVAL);
2071
2072         instr->type = INSTR_HDR_VALIDATE;
2073         instr->valid.header_id = h->id;
2074         return 0;
2075 }
2076
2077 static inline void
2078 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2079 {
2080         struct thread *t = &p->threads[p->thread_id];
2081         struct instruction *ip = t->ip;
2082         uint32_t header_id = ip->valid.header_id;
2083
2084         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2085
2086         /* Headers. */
2087         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2088
2089         /* Thread. */
2090         thread_ip_inc(p);
2091 }
2092
2093 /*
2094  * invalidate.
2095  */
2096 static int
2097 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2098                                struct action *action __rte_unused,
2099                                char **tokens,
2100                                int n_tokens,
2101                                struct instruction *instr,
2102                                struct instruction_data *data __rte_unused)
2103 {
2104         struct header *h;
2105
2106         CHECK(n_tokens == 2, EINVAL);
2107
2108         h = header_parse(p, tokens[1]);
2109         CHECK(h, EINVAL);
2110
2111         instr->type = INSTR_HDR_INVALIDATE;
2112         instr->valid.header_id = h->id;
2113         return 0;
2114 }
2115
2116 static inline void
2117 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2118 {
2119         struct thread *t = &p->threads[p->thread_id];
2120         struct instruction *ip = t->ip;
2121         uint32_t header_id = ip->valid.header_id;
2122
2123         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2124
2125         /* Headers. */
2126         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2127
2128         /* Thread. */
2129         thread_ip_inc(p);
2130 }
2131
2132 /*
2133  * table.
2134  */
2135 static struct table *
2136 table_find(struct rte_swx_pipeline *p, const char *name);
2137
2138 static struct selector *
2139 selector_find(struct rte_swx_pipeline *p, const char *name);
2140
2141 static struct learner *
2142 learner_find(struct rte_swx_pipeline *p, const char *name);
2143
2144 static int
2145 instr_table_translate(struct rte_swx_pipeline *p,
2146                       struct action *action,
2147                       char **tokens,
2148                       int n_tokens,
2149                       struct instruction *instr,
2150                       struct instruction_data *data __rte_unused)
2151 {
2152         struct table *t;
2153         struct selector *s;
2154         struct learner *l;
2155
2156         CHECK(!action, EINVAL);
2157         CHECK(n_tokens == 2, EINVAL);
2158
2159         t = table_find(p, tokens[1]);
2160         if (t) {
2161                 instr->type = INSTR_TABLE;
2162                 instr->table.table_id = t->id;
2163                 return 0;
2164         }
2165
2166         s = selector_find(p, tokens[1]);
2167         if (s) {
2168                 instr->type = INSTR_SELECTOR;
2169                 instr->table.table_id = s->id;
2170                 return 0;
2171         }
2172
2173         l = learner_find(p, tokens[1]);
2174         if (l) {
2175                 instr->type = INSTR_LEARNER;
2176                 instr->table.table_id = l->id;
2177                 return 0;
2178         }
2179
2180         CHECK(0, EINVAL);
2181 }
2182
2183 static inline void
2184 instr_table_exec(struct rte_swx_pipeline *p)
2185 {
2186         struct thread *t = &p->threads[p->thread_id];
2187         struct instruction *ip = t->ip;
2188         uint32_t table_id = ip->table.table_id;
2189         struct rte_swx_table_state *ts = &t->table_state[table_id];
2190         struct table_runtime *table = &t->tables[table_id];
2191         struct table_statistics *stats = &p->table_stats[table_id];
2192         uint64_t action_id, n_pkts_hit, n_pkts_action;
2193         uint8_t *action_data;
2194         int done, hit;
2195
2196         /* Table. */
2197         done = table->func(ts->obj,
2198                            table->mailbox,
2199                            table->key,
2200                            &action_id,
2201                            &action_data,
2202                            &hit);
2203         if (!done) {
2204                 /* Thread. */
2205                 TRACE("[Thread %2u] table %u (not finalized)\n",
2206                       p->thread_id,
2207                       table_id);
2208
2209                 thread_yield(p);
2210                 return;
2211         }
2212
2213         action_id = hit ? action_id : ts->default_action_id;
2214         action_data = hit ? action_data : ts->default_action_data;
2215         n_pkts_hit = stats->n_pkts_hit[hit];
2216         n_pkts_action = stats->n_pkts_action[action_id];
2217
2218         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2219               p->thread_id,
2220               table_id,
2221               hit ? "hit" : "miss",
2222               (uint32_t)action_id);
2223
2224         t->action_id = action_id;
2225         t->structs[0] = action_data;
2226         t->hit = hit;
2227         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2228         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2229
2230         /* Thread. */
2231         thread_ip_action_call(p, t, action_id);
2232 }
2233
2234 static inline void
2235 instr_selector_exec(struct rte_swx_pipeline *p)
2236 {
2237         struct thread *t = &p->threads[p->thread_id];
2238         struct instruction *ip = t->ip;
2239         uint32_t selector_id = ip->table.table_id;
2240         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2241         struct selector_runtime *selector = &t->selectors[selector_id];
2242         struct selector_statistics *stats = &p->selector_stats[selector_id];
2243         uint64_t n_pkts = stats->n_pkts;
2244         int done;
2245
2246         /* Table. */
2247         done = rte_swx_table_selector_select(ts->obj,
2248                            selector->mailbox,
2249                            selector->group_id_buffer,
2250                            selector->selector_buffer,
2251                            selector->member_id_buffer);
2252         if (!done) {
2253                 /* Thread. */
2254                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2255                       p->thread_id,
2256                       selector_id);
2257
2258                 thread_yield(p);
2259                 return;
2260         }
2261
2262
2263         TRACE("[Thread %2u] selector %u\n",
2264               p->thread_id,
2265               selector_id);
2266
2267         stats->n_pkts = n_pkts + 1;
2268
2269         /* Thread. */
2270         thread_ip_inc(p);
2271 }
2272
2273 static inline void
2274 instr_learner_exec(struct rte_swx_pipeline *p)
2275 {
2276         struct thread *t = &p->threads[p->thread_id];
2277         struct instruction *ip = t->ip;
2278         uint32_t learner_id = ip->table.table_id;
2279         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2280                 p->n_selectors + learner_id];
2281         struct learner_runtime *l = &t->learners[learner_id];
2282         struct learner_statistics *stats = &p->learner_stats[learner_id];
2283         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2284         uint8_t *action_data;
2285         int done, hit;
2286
2287         /* Table. */
2288         time = rte_get_tsc_cycles();
2289
2290         done = rte_swx_table_learner_lookup(ts->obj,
2291                                             l->mailbox,
2292                                             time,
2293                                             l->key,
2294                                             &action_id,
2295                                             &action_data,
2296                                             &hit);
2297         if (!done) {
2298                 /* Thread. */
2299                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2300                       p->thread_id,
2301                       learner_id);
2302
2303                 thread_yield(p);
2304                 return;
2305         }
2306
2307         action_id = hit ? action_id : ts->default_action_id;
2308         action_data = hit ? action_data : ts->default_action_data;
2309         n_pkts_hit = stats->n_pkts_hit[hit];
2310         n_pkts_action = stats->n_pkts_action[action_id];
2311
2312         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2313               p->thread_id,
2314               learner_id,
2315               hit ? "hit" : "miss",
2316               (uint32_t)action_id);
2317
2318         t->action_id = action_id;
2319         t->structs[0] = action_data;
2320         t->hit = hit;
2321         t->learner_id = learner_id;
2322         t->time = time;
2323         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2324         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2325
2326         /* Thread. */
2327         thread_ip_action_call(p, t, action_id);
2328 }
2329
2330 /*
2331  * learn.
2332  */
2333 static struct action *
2334 action_find(struct rte_swx_pipeline *p, const char *name);
2335
2336 static int
2337 action_has_nbo_args(struct action *a);
2338
2339 static int
2340 instr_learn_translate(struct rte_swx_pipeline *p,
2341                       struct action *action,
2342                       char **tokens,
2343                       int n_tokens,
2344                       struct instruction *instr,
2345                       struct instruction_data *data __rte_unused)
2346 {
2347         struct action *a;
2348
2349         CHECK(action, EINVAL);
2350         CHECK(n_tokens == 2, EINVAL);
2351
2352         a = action_find(p, tokens[1]);
2353         CHECK(a, EINVAL);
2354         CHECK(!action_has_nbo_args(a), EINVAL);
2355
2356         instr->type = INSTR_LEARNER_LEARN;
2357         instr->learn.action_id = a->id;
2358
2359         return 0;
2360 }
2361
2362 static inline void
2363 instr_learn_exec(struct rte_swx_pipeline *p)
2364 {
2365         struct thread *t = &p->threads[p->thread_id];
2366         struct instruction *ip = t->ip;
2367         uint64_t action_id = ip->learn.action_id;
2368         uint32_t learner_id = t->learner_id;
2369         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2370                 p->n_selectors + learner_id];
2371         struct learner_runtime *l = &t->learners[learner_id];
2372         struct learner_statistics *stats = &p->learner_stats[learner_id];
2373         uint32_t status;
2374
2375         /* Table. */
2376         status = rte_swx_table_learner_add(ts->obj,
2377                                            l->mailbox,
2378                                            t->time,
2379                                            action_id,
2380                                            l->action_data[action_id]);
2381
2382         TRACE("[Thread %2u] learner %u learn %s\n",
2383               p->thread_id,
2384               learner_id,
2385               status ? "ok" : "error");
2386
2387         stats->n_pkts_learn[status] += 1;
2388
2389         /* Thread. */
2390         thread_ip_inc(p);
2391 }
2392
2393 /*
2394  * forget.
2395  */
2396 static int
2397 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2398                        struct action *action,
2399                        char **tokens __rte_unused,
2400                        int n_tokens,
2401                        struct instruction *instr,
2402                        struct instruction_data *data __rte_unused)
2403 {
2404         CHECK(action, EINVAL);
2405         CHECK(n_tokens == 1, EINVAL);
2406
2407         instr->type = INSTR_LEARNER_FORGET;
2408
2409         return 0;
2410 }
2411
2412 static inline void
2413 instr_forget_exec(struct rte_swx_pipeline *p)
2414 {
2415         struct thread *t = &p->threads[p->thread_id];
2416         uint32_t learner_id = t->learner_id;
2417         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2418                 p->n_selectors + learner_id];
2419         struct learner_runtime *l = &t->learners[learner_id];
2420         struct learner_statistics *stats = &p->learner_stats[learner_id];
2421
2422         /* Table. */
2423         rte_swx_table_learner_delete(ts->obj, l->mailbox);
2424
2425         TRACE("[Thread %2u] learner %u forget\n",
2426               p->thread_id,
2427               learner_id);
2428
2429         stats->n_pkts_forget += 1;
2430
2431         /* Thread. */
2432         thread_ip_inc(p);
2433 }
2434
2435 /*
2436  * extern.
2437  */
2438 static int
2439 instr_extern_translate(struct rte_swx_pipeline *p,
2440                        struct action *action __rte_unused,
2441                        char **tokens,
2442                        int n_tokens,
2443                        struct instruction *instr,
2444                        struct instruction_data *data __rte_unused)
2445 {
2446         char *token = tokens[1];
2447
2448         CHECK(n_tokens == 2, EINVAL);
2449
2450         if (token[0] == 'e') {
2451                 struct extern_obj *obj;
2452                 struct extern_type_member_func *func;
2453
2454                 func = extern_obj_member_func_parse(p, token, &obj);
2455                 CHECK(func, EINVAL);
2456
2457                 instr->type = INSTR_EXTERN_OBJ;
2458                 instr->ext_obj.ext_obj_id = obj->id;
2459                 instr->ext_obj.func_id = func->id;
2460
2461                 return 0;
2462         }
2463
2464         if (token[0] == 'f') {
2465                 struct extern_func *func;
2466
2467                 func = extern_func_parse(p, token);
2468                 CHECK(func, EINVAL);
2469
2470                 instr->type = INSTR_EXTERN_FUNC;
2471                 instr->ext_func.ext_func_id = func->id;
2472
2473                 return 0;
2474         }
2475
2476         CHECK(0, EINVAL);
2477 }
2478
2479 static inline void
2480 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2481 {
2482         struct thread *t = &p->threads[p->thread_id];
2483         struct instruction *ip = t->ip;
2484         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2485         uint32_t func_id = ip->ext_obj.func_id;
2486         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2487         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2488
2489         TRACE("[Thread %2u] extern obj %u member func %u\n",
2490               p->thread_id,
2491               obj_id,
2492               func_id);
2493
2494         /* Extern object member function execute. */
2495         uint32_t done = func(obj->obj, obj->mailbox);
2496
2497         /* Thread. */
2498         thread_ip_inc_cond(t, done);
2499         thread_yield_cond(p, done ^ 1);
2500 }
2501
2502 static inline void
2503 instr_extern_func_exec(struct rte_swx_pipeline *p)
2504 {
2505         struct thread *t = &p->threads[p->thread_id];
2506         struct instruction *ip = t->ip;
2507         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2508         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2509         rte_swx_extern_func_t func = ext_func->func;
2510
2511         TRACE("[Thread %2u] extern func %u\n",
2512               p->thread_id,
2513               ext_func_id);
2514
2515         /* Extern function execute. */
2516         uint32_t done = func(ext_func->mailbox);
2517
2518         /* Thread. */
2519         thread_ip_inc_cond(t, done);
2520         thread_yield_cond(p, done ^ 1);
2521 }
2522
2523 /*
2524  * mov.
2525  */
2526 static int
2527 instr_mov_translate(struct rte_swx_pipeline *p,
2528                     struct action *action,
2529                     char **tokens,
2530                     int n_tokens,
2531                     struct instruction *instr,
2532                     struct instruction_data *data __rte_unused)
2533 {
2534         char *dst = tokens[1], *src = tokens[2];
2535         struct field *fdst, *fsrc;
2536         uint64_t src_val;
2537         uint32_t dst_struct_id = 0, src_struct_id = 0;
2538
2539         CHECK(n_tokens == 3, EINVAL);
2540
2541         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2542         CHECK(fdst, EINVAL);
2543         CHECK(!fdst->var_size, EINVAL);
2544
2545         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2546         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2547         if (fsrc) {
2548                 CHECK(!fsrc->var_size, EINVAL);
2549
2550                 instr->type = INSTR_MOV;
2551                 if (dst[0] != 'h' && src[0] == 'h')
2552                         instr->type = INSTR_MOV_MH;
2553                 if (dst[0] == 'h' && src[0] != 'h')
2554                         instr->type = INSTR_MOV_HM;
2555                 if (dst[0] == 'h' && src[0] == 'h')
2556                         instr->type = INSTR_MOV_HH;
2557
2558                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2559                 instr->mov.dst.n_bits = fdst->n_bits;
2560                 instr->mov.dst.offset = fdst->offset / 8;
2561                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2562                 instr->mov.src.n_bits = fsrc->n_bits;
2563                 instr->mov.src.offset = fsrc->offset / 8;
2564                 return 0;
2565         }
2566
2567         /* MOV_I. */
2568         src_val = strtoull(src, &src, 0);
2569         CHECK(!src[0], EINVAL);
2570
2571         if (dst[0] == 'h')
2572                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2573
2574         instr->type = INSTR_MOV_I;
2575         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2576         instr->mov.dst.n_bits = fdst->n_bits;
2577         instr->mov.dst.offset = fdst->offset / 8;
2578         instr->mov.src_val = src_val;
2579         return 0;
2580 }
2581
2582 static inline void
2583 instr_mov_exec(struct rte_swx_pipeline *p)
2584 {
2585         struct thread *t = &p->threads[p->thread_id];
2586         struct instruction *ip = t->ip;
2587
2588         TRACE("[Thread %2u] mov\n",
2589               p->thread_id);
2590
2591         MOV(t, ip);
2592
2593         /* Thread. */
2594         thread_ip_inc(p);
2595 }
2596
2597 static inline void
2598 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2599 {
2600         struct thread *t = &p->threads[p->thread_id];
2601         struct instruction *ip = t->ip;
2602
2603         TRACE("[Thread %2u] mov (mh)\n",
2604               p->thread_id);
2605
2606         MOV_MH(t, ip);
2607
2608         /* Thread. */
2609         thread_ip_inc(p);
2610 }
2611
2612 static inline void
2613 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2614 {
2615         struct thread *t = &p->threads[p->thread_id];
2616         struct instruction *ip = t->ip;
2617
2618         TRACE("[Thread %2u] mov (hm)\n",
2619               p->thread_id);
2620
2621         MOV_HM(t, ip);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 static inline void
2628 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2629 {
2630         struct thread *t = &p->threads[p->thread_id];
2631         struct instruction *ip = t->ip;
2632
2633         TRACE("[Thread %2u] mov (hh)\n",
2634               p->thread_id);
2635
2636         MOV_HH(t, ip);
2637
2638         /* Thread. */
2639         thread_ip_inc(p);
2640 }
2641
2642 static inline void
2643 instr_mov_i_exec(struct rte_swx_pipeline *p)
2644 {
2645         struct thread *t = &p->threads[p->thread_id];
2646         struct instruction *ip = t->ip;
2647
2648         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2649               p->thread_id,
2650               ip->mov.src_val);
2651
2652         MOV_I(t, ip);
2653
2654         /* Thread. */
2655         thread_ip_inc(p);
2656 }
2657
2658 /*
2659  * dma.
2660  */
2661 static inline void
2662 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2663
2664 static inline void
2665 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2666 {
2667         struct thread *t = &p->threads[p->thread_id];
2668         struct instruction *ip = t->ip;
2669         uint8_t *action_data = t->structs[0];
2670         uint64_t valid_headers = t->valid_headers;
2671         uint32_t i;
2672
2673         for (i = 0; i < n_dma; i++) {
2674                 uint32_t header_id = ip->dma.dst.header_id[i];
2675                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2676                 uint32_t offset = ip->dma.src.offset[i];
2677                 uint32_t n_bytes = ip->dma.n_bytes[i];
2678
2679                 struct header_runtime *h = &t->headers[header_id];
2680                 uint8_t *h_ptr0 = h->ptr0;
2681                 uint8_t *h_ptr = t->structs[struct_id];
2682
2683                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2684                         h_ptr : h_ptr0;
2685                 void *src = &action_data[offset];
2686
2687                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2688
2689                 /* Headers. */
2690                 memcpy(dst, src, n_bytes);
2691                 t->structs[struct_id] = dst;
2692                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2693         }
2694
2695         t->valid_headers = valid_headers;
2696 }
2697
2698 static inline void
2699 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2700 {
2701         __instr_dma_ht_exec(p, 1);
2702
2703         /* Thread. */
2704         thread_ip_inc(p);
2705 }
2706
2707 static inline void
2708 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2709 {
2710         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2711               p->thread_id);
2712
2713         __instr_dma_ht_exec(p, 2);
2714
2715         /* Thread. */
2716         thread_ip_inc(p);
2717 }
2718
2719 static inline void
2720 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2721 {
2722         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2723               p->thread_id);
2724
2725         __instr_dma_ht_exec(p, 3);
2726
2727         /* Thread. */
2728         thread_ip_inc(p);
2729 }
2730
2731 static inline void
2732 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2733 {
2734         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2735               p->thread_id);
2736
2737         __instr_dma_ht_exec(p, 4);
2738
2739         /* Thread. */
2740         thread_ip_inc(p);
2741 }
2742
2743 static inline void
2744 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2745 {
2746         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2747               p->thread_id);
2748
2749         __instr_dma_ht_exec(p, 5);
2750
2751         /* Thread. */
2752         thread_ip_inc(p);
2753 }
2754
2755 static inline void
2756 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2757 {
2758         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2759               p->thread_id);
2760
2761         __instr_dma_ht_exec(p, 6);
2762
2763         /* Thread. */
2764         thread_ip_inc(p);
2765 }
2766
2767 static inline void
2768 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2769 {
2770         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2771               p->thread_id);
2772
2773         __instr_dma_ht_exec(p, 7);
2774
2775         /* Thread. */
2776         thread_ip_inc(p);
2777 }
2778
2779 static inline void
2780 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2781 {
2782         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2783               p->thread_id);
2784
2785         __instr_dma_ht_exec(p, 8);
2786
2787         /* Thread. */
2788         thread_ip_inc(p);
2789 }
2790
2791 /*
2792  * alu.
2793  */
2794 static int
2795 instr_alu_add_translate(struct rte_swx_pipeline *p,
2796                         struct action *action,
2797                         char **tokens,
2798                         int n_tokens,
2799                         struct instruction *instr,
2800                         struct instruction_data *data __rte_unused)
2801 {
2802         char *dst = tokens[1], *src = tokens[2];
2803         struct field *fdst, *fsrc;
2804         uint64_t src_val;
2805         uint32_t dst_struct_id = 0, src_struct_id = 0;
2806
2807         CHECK(n_tokens == 3, EINVAL);
2808
2809         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2810         CHECK(fdst, EINVAL);
2811         CHECK(!fdst->var_size, EINVAL);
2812
2813         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2814         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2815         if (fsrc) {
2816                 CHECK(!fsrc->var_size, EINVAL);
2817
2818                 instr->type = INSTR_ALU_ADD;
2819                 if (dst[0] == 'h' && src[0] != 'h')
2820                         instr->type = INSTR_ALU_ADD_HM;
2821                 if (dst[0] != 'h' && src[0] == 'h')
2822                         instr->type = INSTR_ALU_ADD_MH;
2823                 if (dst[0] == 'h' && src[0] == 'h')
2824                         instr->type = INSTR_ALU_ADD_HH;
2825
2826                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2827                 instr->alu.dst.n_bits = fdst->n_bits;
2828                 instr->alu.dst.offset = fdst->offset / 8;
2829                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2830                 instr->alu.src.n_bits = fsrc->n_bits;
2831                 instr->alu.src.offset = fsrc->offset / 8;
2832                 return 0;
2833         }
2834
2835         /* ADD_MI, ADD_HI. */
2836         src_val = strtoull(src, &src, 0);
2837         CHECK(!src[0], EINVAL);
2838
2839         instr->type = INSTR_ALU_ADD_MI;
2840         if (dst[0] == 'h')
2841                 instr->type = INSTR_ALU_ADD_HI;
2842
2843         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2844         instr->alu.dst.n_bits = fdst->n_bits;
2845         instr->alu.dst.offset = fdst->offset / 8;
2846         instr->alu.src_val = src_val;
2847         return 0;
2848 }
2849
2850 static int
2851 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2852                         struct action *action,
2853                         char **tokens,
2854                         int n_tokens,
2855                         struct instruction *instr,
2856                         struct instruction_data *data __rte_unused)
2857 {
2858         char *dst = tokens[1], *src = tokens[2];
2859         struct field *fdst, *fsrc;
2860         uint64_t src_val;
2861         uint32_t dst_struct_id = 0, src_struct_id = 0;
2862
2863         CHECK(n_tokens == 3, EINVAL);
2864
2865         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2866         CHECK(fdst, EINVAL);
2867         CHECK(!fdst->var_size, EINVAL);
2868
2869         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2870         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2871         if (fsrc) {
2872                 CHECK(!fsrc->var_size, EINVAL);
2873
2874                 instr->type = INSTR_ALU_SUB;
2875                 if (dst[0] == 'h' && src[0] != 'h')
2876                         instr->type = INSTR_ALU_SUB_HM;
2877                 if (dst[0] != 'h' && src[0] == 'h')
2878                         instr->type = INSTR_ALU_SUB_MH;
2879                 if (dst[0] == 'h' && src[0] == 'h')
2880                         instr->type = INSTR_ALU_SUB_HH;
2881
2882                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2883                 instr->alu.dst.n_bits = fdst->n_bits;
2884                 instr->alu.dst.offset = fdst->offset / 8;
2885                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2886                 instr->alu.src.n_bits = fsrc->n_bits;
2887                 instr->alu.src.offset = fsrc->offset / 8;
2888                 return 0;
2889         }
2890
2891         /* SUB_MI, SUB_HI. */
2892         src_val = strtoull(src, &src, 0);
2893         CHECK(!src[0], EINVAL);
2894
2895         instr->type = INSTR_ALU_SUB_MI;
2896         if (dst[0] == 'h')
2897                 instr->type = INSTR_ALU_SUB_HI;
2898
2899         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2900         instr->alu.dst.n_bits = fdst->n_bits;
2901         instr->alu.dst.offset = fdst->offset / 8;
2902         instr->alu.src_val = src_val;
2903         return 0;
2904 }
2905
2906 static int
2907 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2908                           struct action *action __rte_unused,
2909                           char **tokens,
2910                           int n_tokens,
2911                           struct instruction *instr,
2912                           struct instruction_data *data __rte_unused)
2913 {
2914         char *dst = tokens[1], *src = tokens[2];
2915         struct header *hdst, *hsrc;
2916         struct field *fdst, *fsrc;
2917
2918         CHECK(n_tokens == 3, EINVAL);
2919
2920         fdst = header_field_parse(p, dst, &hdst);
2921         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2922         CHECK(!fdst->var_size, EINVAL);
2923
2924         /* CKADD_FIELD. */
2925         fsrc = header_field_parse(p, src, &hsrc);
2926         if (fsrc) {
2927                 CHECK(!fsrc->var_size, EINVAL);
2928
2929                 instr->type = INSTR_ALU_CKADD_FIELD;
2930                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2931                 instr->alu.dst.n_bits = fdst->n_bits;
2932                 instr->alu.dst.offset = fdst->offset / 8;
2933                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2934                 instr->alu.src.n_bits = fsrc->n_bits;
2935                 instr->alu.src.offset = fsrc->offset / 8;
2936                 return 0;
2937         }
2938
2939         /* CKADD_STRUCT, CKADD_STRUCT20. */
2940         hsrc = header_parse(p, src);
2941         CHECK(hsrc, EINVAL);
2942         CHECK(!hsrc->st->var_size, EINVAL);
2943
2944         instr->type = INSTR_ALU_CKADD_STRUCT;
2945         if ((hsrc->st->n_bits / 8) == 20)
2946                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2947
2948         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2949         instr->alu.dst.n_bits = fdst->n_bits;
2950         instr->alu.dst.offset = fdst->offset / 8;
2951         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2952         instr->alu.src.n_bits = hsrc->st->n_bits;
2953         instr->alu.src.offset = 0; /* Unused. */
2954         return 0;
2955 }
2956
2957 static int
2958 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2959                           struct action *action __rte_unused,
2960                           char **tokens,
2961                           int n_tokens,
2962                           struct instruction *instr,
2963                           struct instruction_data *data __rte_unused)
2964 {
2965         char *dst = tokens[1], *src = tokens[2];
2966         struct header *hdst, *hsrc;
2967         struct field *fdst, *fsrc;
2968
2969         CHECK(n_tokens == 3, EINVAL);
2970
2971         fdst = header_field_parse(p, dst, &hdst);
2972         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2973         CHECK(!fdst->var_size, EINVAL);
2974
2975         fsrc = header_field_parse(p, src, &hsrc);
2976         CHECK(fsrc, EINVAL);
2977         CHECK(!fsrc->var_size, EINVAL);
2978
2979         instr->type = INSTR_ALU_CKSUB_FIELD;
2980         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2981         instr->alu.dst.n_bits = fdst->n_bits;
2982         instr->alu.dst.offset = fdst->offset / 8;
2983         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2984         instr->alu.src.n_bits = fsrc->n_bits;
2985         instr->alu.src.offset = fsrc->offset / 8;
2986         return 0;
2987 }
2988
2989 static int
2990 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2991                         struct action *action,
2992                         char **tokens,
2993                         int n_tokens,
2994                         struct instruction *instr,
2995                         struct instruction_data *data __rte_unused)
2996 {
2997         char *dst = tokens[1], *src = tokens[2];
2998         struct field *fdst, *fsrc;
2999         uint64_t src_val;
3000         uint32_t dst_struct_id = 0, src_struct_id = 0;
3001
3002         CHECK(n_tokens == 3, EINVAL);
3003
3004         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3005         CHECK(fdst, EINVAL);
3006         CHECK(!fdst->var_size, EINVAL);
3007
3008         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3009         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3010         if (fsrc) {
3011                 CHECK(!fsrc->var_size, EINVAL);
3012
3013                 instr->type = INSTR_ALU_SHL;
3014                 if (dst[0] == 'h' && src[0] != 'h')
3015                         instr->type = INSTR_ALU_SHL_HM;
3016                 if (dst[0] != 'h' && src[0] == 'h')
3017                         instr->type = INSTR_ALU_SHL_MH;
3018                 if (dst[0] == 'h' && src[0] == 'h')
3019                         instr->type = INSTR_ALU_SHL_HH;
3020
3021                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3022                 instr->alu.dst.n_bits = fdst->n_bits;
3023                 instr->alu.dst.offset = fdst->offset / 8;
3024                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3025                 instr->alu.src.n_bits = fsrc->n_bits;
3026                 instr->alu.src.offset = fsrc->offset / 8;
3027                 return 0;
3028         }
3029
3030         /* SHL_MI, SHL_HI. */
3031         src_val = strtoull(src, &src, 0);
3032         CHECK(!src[0], EINVAL);
3033
3034         instr->type = INSTR_ALU_SHL_MI;
3035         if (dst[0] == 'h')
3036                 instr->type = INSTR_ALU_SHL_HI;
3037
3038         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3039         instr->alu.dst.n_bits = fdst->n_bits;
3040         instr->alu.dst.offset = fdst->offset / 8;
3041         instr->alu.src_val = src_val;
3042         return 0;
3043 }
3044
3045 static int
3046 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3047                         struct action *action,
3048                         char **tokens,
3049                         int n_tokens,
3050                         struct instruction *instr,
3051                         struct instruction_data *data __rte_unused)
3052 {
3053         char *dst = tokens[1], *src = tokens[2];
3054         struct field *fdst, *fsrc;
3055         uint64_t src_val;
3056         uint32_t dst_struct_id = 0, src_struct_id = 0;
3057
3058         CHECK(n_tokens == 3, EINVAL);
3059
3060         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3061         CHECK(fdst, EINVAL);
3062         CHECK(!fdst->var_size, EINVAL);
3063
3064         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3065         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3066         if (fsrc) {
3067                 CHECK(!fsrc->var_size, EINVAL);
3068
3069                 instr->type = INSTR_ALU_SHR;
3070                 if (dst[0] == 'h' && src[0] != 'h')
3071                         instr->type = INSTR_ALU_SHR_HM;
3072                 if (dst[0] != 'h' && src[0] == 'h')
3073                         instr->type = INSTR_ALU_SHR_MH;
3074                 if (dst[0] == 'h' && src[0] == 'h')
3075                         instr->type = INSTR_ALU_SHR_HH;
3076
3077                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3078                 instr->alu.dst.n_bits = fdst->n_bits;
3079                 instr->alu.dst.offset = fdst->offset / 8;
3080                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3081                 instr->alu.src.n_bits = fsrc->n_bits;
3082                 instr->alu.src.offset = fsrc->offset / 8;
3083                 return 0;
3084         }
3085
3086         /* SHR_MI, SHR_HI. */
3087         src_val = strtoull(src, &src, 0);
3088         CHECK(!src[0], EINVAL);
3089
3090         instr->type = INSTR_ALU_SHR_MI;
3091         if (dst[0] == 'h')
3092                 instr->type = INSTR_ALU_SHR_HI;
3093
3094         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3095         instr->alu.dst.n_bits = fdst->n_bits;
3096         instr->alu.dst.offset = fdst->offset / 8;
3097         instr->alu.src_val = src_val;
3098         return 0;
3099 }
3100
3101 static int
3102 instr_alu_and_translate(struct rte_swx_pipeline *p,
3103                         struct action *action,
3104                         char **tokens,
3105                         int n_tokens,
3106                         struct instruction *instr,
3107                         struct instruction_data *data __rte_unused)
3108 {
3109         char *dst = tokens[1], *src = tokens[2];
3110         struct field *fdst, *fsrc;
3111         uint64_t src_val;
3112         uint32_t dst_struct_id = 0, src_struct_id = 0;
3113
3114         CHECK(n_tokens == 3, EINVAL);
3115
3116         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3117         CHECK(fdst, EINVAL);
3118         CHECK(!fdst->var_size, EINVAL);
3119
3120         /* AND, AND_MH, AND_HM, AND_HH. */
3121         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3122         if (fsrc) {
3123                 CHECK(!fsrc->var_size, EINVAL);
3124
3125                 instr->type = INSTR_ALU_AND;
3126                 if (dst[0] != 'h' && src[0] == 'h')
3127                         instr->type = INSTR_ALU_AND_MH;
3128                 if (dst[0] == 'h' && src[0] != 'h')
3129                         instr->type = INSTR_ALU_AND_HM;
3130                 if (dst[0] == 'h' && src[0] == 'h')
3131                         instr->type = INSTR_ALU_AND_HH;
3132
3133                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3134                 instr->alu.dst.n_bits = fdst->n_bits;
3135                 instr->alu.dst.offset = fdst->offset / 8;
3136                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3137                 instr->alu.src.n_bits = fsrc->n_bits;
3138                 instr->alu.src.offset = fsrc->offset / 8;
3139                 return 0;
3140         }
3141
3142         /* AND_I. */
3143         src_val = strtoull(src, &src, 0);
3144         CHECK(!src[0], EINVAL);
3145
3146         if (dst[0] == 'h')
3147                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3148
3149         instr->type = INSTR_ALU_AND_I;
3150         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3151         instr->alu.dst.n_bits = fdst->n_bits;
3152         instr->alu.dst.offset = fdst->offset / 8;
3153         instr->alu.src_val = src_val;
3154         return 0;
3155 }
3156
3157 static int
3158 instr_alu_or_translate(struct rte_swx_pipeline *p,
3159                        struct action *action,
3160                        char **tokens,
3161                        int n_tokens,
3162                        struct instruction *instr,
3163                        struct instruction_data *data __rte_unused)
3164 {
3165         char *dst = tokens[1], *src = tokens[2];
3166         struct field *fdst, *fsrc;
3167         uint64_t src_val;
3168         uint32_t dst_struct_id = 0, src_struct_id = 0;
3169
3170         CHECK(n_tokens == 3, EINVAL);
3171
3172         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3173         CHECK(fdst, EINVAL);
3174         CHECK(!fdst->var_size, EINVAL);
3175
3176         /* OR, OR_MH, OR_HM, OR_HH. */
3177         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3178         if (fsrc) {
3179                 CHECK(!fsrc->var_size, EINVAL);
3180
3181                 instr->type = INSTR_ALU_OR;
3182                 if (dst[0] != 'h' && src[0] == 'h')
3183                         instr->type = INSTR_ALU_OR_MH;
3184                 if (dst[0] == 'h' && src[0] != 'h')
3185                         instr->type = INSTR_ALU_OR_HM;
3186                 if (dst[0] == 'h' && src[0] == 'h')
3187                         instr->type = INSTR_ALU_OR_HH;
3188
3189                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3190                 instr->alu.dst.n_bits = fdst->n_bits;
3191                 instr->alu.dst.offset = fdst->offset / 8;
3192                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3193                 instr->alu.src.n_bits = fsrc->n_bits;
3194                 instr->alu.src.offset = fsrc->offset / 8;
3195                 return 0;
3196         }
3197
3198         /* OR_I. */
3199         src_val = strtoull(src, &src, 0);
3200         CHECK(!src[0], EINVAL);
3201
3202         if (dst[0] == 'h')
3203                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3204
3205         instr->type = INSTR_ALU_OR_I;
3206         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3207         instr->alu.dst.n_bits = fdst->n_bits;
3208         instr->alu.dst.offset = fdst->offset / 8;
3209         instr->alu.src_val = src_val;
3210         return 0;
3211 }
3212
3213 static int
3214 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3215                         struct action *action,
3216                         char **tokens,
3217                         int n_tokens,
3218                         struct instruction *instr,
3219                         struct instruction_data *data __rte_unused)
3220 {
3221         char *dst = tokens[1], *src = tokens[2];
3222         struct field *fdst, *fsrc;
3223         uint64_t src_val;
3224         uint32_t dst_struct_id = 0, src_struct_id = 0;
3225
3226         CHECK(n_tokens == 3, EINVAL);
3227
3228         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3229         CHECK(fdst, EINVAL);
3230         CHECK(!fdst->var_size, EINVAL);
3231
3232         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3233         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3234         if (fsrc) {
3235                 CHECK(!fsrc->var_size, EINVAL);
3236
3237                 instr->type = INSTR_ALU_XOR;
3238                 if (dst[0] != 'h' && src[0] == 'h')
3239                         instr->type = INSTR_ALU_XOR_MH;
3240                 if (dst[0] == 'h' && src[0] != 'h')
3241                         instr->type = INSTR_ALU_XOR_HM;
3242                 if (dst[0] == 'h' && src[0] == 'h')
3243                         instr->type = INSTR_ALU_XOR_HH;
3244
3245                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3246                 instr->alu.dst.n_bits = fdst->n_bits;
3247                 instr->alu.dst.offset = fdst->offset / 8;
3248                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3249                 instr->alu.src.n_bits = fsrc->n_bits;
3250                 instr->alu.src.offset = fsrc->offset / 8;
3251                 return 0;
3252         }
3253
3254         /* XOR_I. */
3255         src_val = strtoull(src, &src, 0);
3256         CHECK(!src[0], EINVAL);
3257
3258         if (dst[0] == 'h')
3259                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3260
3261         instr->type = INSTR_ALU_XOR_I;
3262         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3263         instr->alu.dst.n_bits = fdst->n_bits;
3264         instr->alu.dst.offset = fdst->offset / 8;
3265         instr->alu.src_val = src_val;
3266         return 0;
3267 }
3268
3269 static inline void
3270 instr_alu_add_exec(struct rte_swx_pipeline *p)
3271 {
3272         struct thread *t = &p->threads[p->thread_id];
3273         struct instruction *ip = t->ip;
3274
3275         TRACE("[Thread %2u] add\n", p->thread_id);
3276
3277         /* Structs. */
3278         ALU(t, ip, +);
3279
3280         /* Thread. */
3281         thread_ip_inc(p);
3282 }
3283
3284 static inline void
3285 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3286 {
3287         struct thread *t = &p->threads[p->thread_id];
3288         struct instruction *ip = t->ip;
3289
3290         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3291
3292         /* Structs. */
3293         ALU_MH(t, ip, +);
3294
3295         /* Thread. */
3296         thread_ip_inc(p);
3297 }
3298
3299 static inline void
3300 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3301 {
3302         struct thread *t = &p->threads[p->thread_id];
3303         struct instruction *ip = t->ip;
3304
3305         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3306
3307         /* Structs. */
3308         ALU_HM(t, ip, +);
3309
3310         /* Thread. */
3311         thread_ip_inc(p);
3312 }
3313
3314 static inline void
3315 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3316 {
3317         struct thread *t = &p->threads[p->thread_id];
3318         struct instruction *ip = t->ip;
3319
3320         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3321
3322         /* Structs. */
3323         ALU_HH(t, ip, +);
3324
3325         /* Thread. */
3326         thread_ip_inc(p);
3327 }
3328
3329 static inline void
3330 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3331 {
3332         struct thread *t = &p->threads[p->thread_id];
3333         struct instruction *ip = t->ip;
3334
3335         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3336
3337         /* Structs. */
3338         ALU_MI(t, ip, +);
3339
3340         /* Thread. */
3341         thread_ip_inc(p);
3342 }
3343
3344 static inline void
3345 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3346 {
3347         struct thread *t = &p->threads[p->thread_id];
3348         struct instruction *ip = t->ip;
3349
3350         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3351
3352         /* Structs. */
3353         ALU_HI(t, ip, +);
3354
3355         /* Thread. */
3356         thread_ip_inc(p);
3357 }
3358
3359 static inline void
3360 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3361 {
3362         struct thread *t = &p->threads[p->thread_id];
3363         struct instruction *ip = t->ip;
3364
3365         TRACE("[Thread %2u] sub\n", p->thread_id);
3366
3367         /* Structs. */
3368         ALU(t, ip, -);
3369
3370         /* Thread. */
3371         thread_ip_inc(p);
3372 }
3373
3374 static inline void
3375 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3376 {
3377         struct thread *t = &p->threads[p->thread_id];
3378         struct instruction *ip = t->ip;
3379
3380         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3381
3382         /* Structs. */
3383         ALU_MH(t, ip, -);
3384
3385         /* Thread. */
3386         thread_ip_inc(p);
3387 }
3388
3389 static inline void
3390 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3391 {
3392         struct thread *t = &p->threads[p->thread_id];
3393         struct instruction *ip = t->ip;
3394
3395         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3396
3397         /* Structs. */
3398         ALU_HM(t, ip, -);
3399
3400         /* Thread. */
3401         thread_ip_inc(p);
3402 }
3403
3404 static inline void
3405 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3406 {
3407         struct thread *t = &p->threads[p->thread_id];
3408         struct instruction *ip = t->ip;
3409
3410         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3411
3412         /* Structs. */
3413         ALU_HH(t, ip, -);
3414
3415         /* Thread. */
3416         thread_ip_inc(p);
3417 }
3418
3419 static inline void
3420 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3421 {
3422         struct thread *t = &p->threads[p->thread_id];
3423         struct instruction *ip = t->ip;
3424
3425         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3426
3427         /* Structs. */
3428         ALU_MI(t, ip, -);
3429
3430         /* Thread. */
3431         thread_ip_inc(p);
3432 }
3433
3434 static inline void
3435 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3436 {
3437         struct thread *t = &p->threads[p->thread_id];
3438         struct instruction *ip = t->ip;
3439
3440         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3441
3442         /* Structs. */
3443         ALU_HI(t, ip, -);
3444
3445         /* Thread. */
3446         thread_ip_inc(p);
3447 }
3448
3449 static inline void
3450 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3451 {
3452         struct thread *t = &p->threads[p->thread_id];
3453         struct instruction *ip = t->ip;
3454
3455         TRACE("[Thread %2u] shl\n", p->thread_id);
3456
3457         /* Structs. */
3458         ALU(t, ip, <<);
3459
3460         /* Thread. */
3461         thread_ip_inc(p);
3462 }
3463
3464 static inline void
3465 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3466 {
3467         struct thread *t = &p->threads[p->thread_id];
3468         struct instruction *ip = t->ip;
3469
3470         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3471
3472         /* Structs. */
3473         ALU_MH(t, ip, <<);
3474
3475         /* Thread. */
3476         thread_ip_inc(p);
3477 }
3478
3479 static inline void
3480 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3481 {
3482         struct thread *t = &p->threads[p->thread_id];
3483         struct instruction *ip = t->ip;
3484
3485         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3486
3487         /* Structs. */
3488         ALU_HM(t, ip, <<);
3489
3490         /* Thread. */
3491         thread_ip_inc(p);
3492 }
3493
3494 static inline void
3495 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3496 {
3497         struct thread *t = &p->threads[p->thread_id];
3498         struct instruction *ip = t->ip;
3499
3500         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3501
3502         /* Structs. */
3503         ALU_HH(t, ip, <<);
3504
3505         /* Thread. */
3506         thread_ip_inc(p);
3507 }
3508
3509 static inline void
3510 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3511 {
3512         struct thread *t = &p->threads[p->thread_id];
3513         struct instruction *ip = t->ip;
3514
3515         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3516
3517         /* Structs. */
3518         ALU_MI(t, ip, <<);
3519
3520         /* Thread. */
3521         thread_ip_inc(p);
3522 }
3523
3524 static inline void
3525 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3526 {
3527         struct thread *t = &p->threads[p->thread_id];
3528         struct instruction *ip = t->ip;
3529
3530         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3531
3532         /* Structs. */
3533         ALU_HI(t, ip, <<);
3534
3535         /* Thread. */
3536         thread_ip_inc(p);
3537 }
3538
3539 static inline void
3540 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3541 {
3542         struct thread *t = &p->threads[p->thread_id];
3543         struct instruction *ip = t->ip;
3544
3545         TRACE("[Thread %2u] shr\n", p->thread_id);
3546
3547         /* Structs. */
3548         ALU(t, ip, >>);
3549
3550         /* Thread. */
3551         thread_ip_inc(p);
3552 }
3553
3554 static inline void
3555 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3556 {
3557         struct thread *t = &p->threads[p->thread_id];
3558         struct instruction *ip = t->ip;
3559
3560         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3561
3562         /* Structs. */
3563         ALU_MH(t, ip, >>);
3564
3565         /* Thread. */
3566         thread_ip_inc(p);
3567 }
3568
3569 static inline void
3570 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3571 {
3572         struct thread *t = &p->threads[p->thread_id];
3573         struct instruction *ip = t->ip;
3574
3575         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3576
3577         /* Structs. */
3578         ALU_HM(t, ip, >>);
3579
3580         /* Thread. */
3581         thread_ip_inc(p);
3582 }
3583
3584 static inline void
3585 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3586 {
3587         struct thread *t = &p->threads[p->thread_id];
3588         struct instruction *ip = t->ip;
3589
3590         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3591
3592         /* Structs. */
3593         ALU_HH(t, ip, >>);
3594
3595         /* Thread. */
3596         thread_ip_inc(p);
3597 }
3598
3599 static inline void
3600 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3601 {
3602         struct thread *t = &p->threads[p->thread_id];
3603         struct instruction *ip = t->ip;
3604
3605         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3606
3607         /* Structs. */
3608         ALU_MI(t, ip, >>);
3609
3610         /* Thread. */
3611         thread_ip_inc(p);
3612 }
3613
3614 static inline void
3615 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3616 {
3617         struct thread *t = &p->threads[p->thread_id];
3618         struct instruction *ip = t->ip;
3619
3620         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3621
3622         /* Structs. */
3623         ALU_HI(t, ip, >>);
3624
3625         /* Thread. */
3626         thread_ip_inc(p);
3627 }
3628
3629 static inline void
3630 instr_alu_and_exec(struct rte_swx_pipeline *p)
3631 {
3632         struct thread *t = &p->threads[p->thread_id];
3633         struct instruction *ip = t->ip;
3634
3635         TRACE("[Thread %2u] and\n", p->thread_id);
3636
3637         /* Structs. */
3638         ALU(t, ip, &);
3639
3640         /* Thread. */
3641         thread_ip_inc(p);
3642 }
3643
3644 static inline void
3645 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3646 {
3647         struct thread *t = &p->threads[p->thread_id];
3648         struct instruction *ip = t->ip;
3649
3650         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3651
3652         /* Structs. */
3653         ALU_MH(t, ip, &);
3654
3655         /* Thread. */
3656         thread_ip_inc(p);
3657 }
3658
3659 static inline void
3660 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3661 {
3662         struct thread *t = &p->threads[p->thread_id];
3663         struct instruction *ip = t->ip;
3664
3665         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3666
3667         /* Structs. */
3668         ALU_HM_FAST(t, ip, &);
3669
3670         /* Thread. */
3671         thread_ip_inc(p);
3672 }
3673
3674 static inline void
3675 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3676 {
3677         struct thread *t = &p->threads[p->thread_id];
3678         struct instruction *ip = t->ip;
3679
3680         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3681
3682         /* Structs. */
3683         ALU_HH_FAST(t, ip, &);
3684
3685         /* Thread. */
3686         thread_ip_inc(p);
3687 }
3688
3689 static inline void
3690 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3691 {
3692         struct thread *t = &p->threads[p->thread_id];
3693         struct instruction *ip = t->ip;
3694
3695         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3696
3697         /* Structs. */
3698         ALU_I(t, ip, &);
3699
3700         /* Thread. */
3701         thread_ip_inc(p);
3702 }
3703
3704 static inline void
3705 instr_alu_or_exec(struct rte_swx_pipeline *p)
3706 {
3707         struct thread *t = &p->threads[p->thread_id];
3708         struct instruction *ip = t->ip;
3709
3710         TRACE("[Thread %2u] or\n", p->thread_id);
3711
3712         /* Structs. */
3713         ALU(t, ip, |);
3714
3715         /* Thread. */
3716         thread_ip_inc(p);
3717 }
3718
3719 static inline void
3720 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3721 {
3722         struct thread *t = &p->threads[p->thread_id];
3723         struct instruction *ip = t->ip;
3724
3725         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3726
3727         /* Structs. */
3728         ALU_MH(t, ip, |);
3729
3730         /* Thread. */
3731         thread_ip_inc(p);
3732 }
3733
3734 static inline void
3735 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3736 {
3737         struct thread *t = &p->threads[p->thread_id];
3738         struct instruction *ip = t->ip;
3739
3740         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3741
3742         /* Structs. */
3743         ALU_HM_FAST(t, ip, |);
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 static inline void
3750 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3751 {
3752         struct thread *t = &p->threads[p->thread_id];
3753         struct instruction *ip = t->ip;
3754
3755         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3756
3757         /* Structs. */
3758         ALU_HH_FAST(t, ip, |);
3759
3760         /* Thread. */
3761         thread_ip_inc(p);
3762 }
3763
3764 static inline void
3765 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3766 {
3767         struct thread *t = &p->threads[p->thread_id];
3768         struct instruction *ip = t->ip;
3769
3770         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3771
3772         /* Structs. */
3773         ALU_I(t, ip, |);
3774
3775         /* Thread. */
3776         thread_ip_inc(p);
3777 }
3778
3779 static inline void
3780 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3781 {
3782         struct thread *t = &p->threads[p->thread_id];
3783         struct instruction *ip = t->ip;
3784
3785         TRACE("[Thread %2u] xor\n", p->thread_id);
3786
3787         /* Structs. */
3788         ALU(t, ip, ^);
3789
3790         /* Thread. */
3791         thread_ip_inc(p);
3792 }
3793
3794 static inline void
3795 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3796 {
3797         struct thread *t = &p->threads[p->thread_id];
3798         struct instruction *ip = t->ip;
3799
3800         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3801
3802         /* Structs. */
3803         ALU_MH(t, ip, ^);
3804
3805         /* Thread. */
3806         thread_ip_inc(p);
3807 }
3808
3809 static inline void
3810 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3811 {
3812         struct thread *t = &p->threads[p->thread_id];
3813         struct instruction *ip = t->ip;
3814
3815         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3816
3817         /* Structs. */
3818         ALU_HM_FAST(t, ip, ^);
3819
3820         /* Thread. */
3821         thread_ip_inc(p);
3822 }
3823
3824 static inline void
3825 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3826 {
3827         struct thread *t = &p->threads[p->thread_id];
3828         struct instruction *ip = t->ip;
3829
3830         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3831
3832         /* Structs. */
3833         ALU_HH_FAST(t, ip, ^);
3834
3835         /* Thread. */
3836         thread_ip_inc(p);
3837 }
3838
3839 static inline void
3840 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3841 {
3842         struct thread *t = &p->threads[p->thread_id];
3843         struct instruction *ip = t->ip;
3844
3845         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3846
3847         /* Structs. */
3848         ALU_I(t, ip, ^);
3849
3850         /* Thread. */
3851         thread_ip_inc(p);
3852 }
3853
3854 static inline void
3855 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3856 {
3857         struct thread *t = &p->threads[p->thread_id];
3858         struct instruction *ip = t->ip;
3859         uint8_t *dst_struct, *src_struct;
3860         uint16_t *dst16_ptr, dst;
3861         uint64_t *src64_ptr, src64, src64_mask, src;
3862         uint64_t r;
3863
3864         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3865
3866         /* Structs. */
3867         dst_struct = t->structs[ip->alu.dst.struct_id];
3868         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3869         dst = *dst16_ptr;
3870
3871         src_struct = t->structs[ip->alu.src.struct_id];
3872         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3873         src64 = *src64_ptr;
3874         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3875         src = src64 & src64_mask;
3876
3877         r = dst;
3878         r = ~r & 0xFFFF;
3879
3880         /* The first input (r) is a 16-bit number. The second and the third
3881          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3882          * three numbers (output r) is a 34-bit number.
3883          */
3884         r += (src >> 32) + (src & 0xFFFFFFFF);
3885
3886         /* The first input is a 16-bit number. The second input is an 18-bit
3887          * number. In the worst case scenario, the sum of the two numbers is a
3888          * 19-bit number.
3889          */
3890         r = (r & 0xFFFF) + (r >> 16);
3891
3892         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3893          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3894          */
3895         r = (r & 0xFFFF) + (r >> 16);
3896
3897         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3898          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3899          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3900          * therefore the output r is always a 16-bit number.
3901          */
3902         r = (r & 0xFFFF) + (r >> 16);
3903
3904         r = ~r & 0xFFFF;
3905         r = r ? r : 0xFFFF;
3906
3907         *dst16_ptr = (uint16_t)r;
3908
3909         /* Thread. */
3910         thread_ip_inc(p);
3911 }
3912
3913 static inline void
3914 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3915 {
3916         struct thread *t = &p->threads[p->thread_id];
3917         struct instruction *ip = t->ip;
3918         uint8_t *dst_struct, *src_struct;
3919         uint16_t *dst16_ptr, dst;
3920         uint64_t *src64_ptr, src64, src64_mask, src;
3921         uint64_t r;
3922
3923         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3924
3925         /* Structs. */
3926         dst_struct = t->structs[ip->alu.dst.struct_id];
3927         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3928         dst = *dst16_ptr;
3929
3930         src_struct = t->structs[ip->alu.src.struct_id];
3931         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3932         src64 = *src64_ptr;
3933         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3934         src = src64 & src64_mask;
3935
3936         r = dst;
3937         r = ~r & 0xFFFF;
3938
3939         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3940          * the following sequence of operations in 2's complement arithmetic:
3941          *    a '- b = (a - b) % 0xFFFF.
3942          *
3943          * In order to prevent an underflow for the below subtraction, in which
3944          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3945          * minuend), we first add a multiple of the 0xFFFF modulus to the
3946          * minuend. The number we add to the minuend needs to be a 34-bit number
3947          * or higher, so for readability reasons we picked the 36-bit multiple.
3948          * We are effectively turning the 16-bit minuend into a 36-bit number:
3949          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3950          */
3951         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3952
3953         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3954          * result (the output r) is a 36-bit number.
3955          */
3956         r -= (src >> 32) + (src & 0xFFFFFFFF);
3957
3958         /* The first input is a 16-bit number. The second input is a 20-bit
3959          * number. Their sum is a 21-bit number.
3960          */
3961         r = (r & 0xFFFF) + (r >> 16);
3962
3963         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3964          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3965          */
3966         r = (r & 0xFFFF) + (r >> 16);
3967
3968         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3969          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3970          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3971          * generated, therefore the output r is always a 16-bit number.
3972          */
3973         r = (r & 0xFFFF) + (r >> 16);
3974
3975         r = ~r & 0xFFFF;
3976         r = r ? r : 0xFFFF;
3977
3978         *dst16_ptr = (uint16_t)r;
3979
3980         /* Thread. */
3981         thread_ip_inc(p);
3982 }
3983
3984 static inline void
3985 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3986 {
3987         struct thread *t = &p->threads[p->thread_id];
3988         struct instruction *ip = t->ip;
3989         uint8_t *dst_struct, *src_struct;
3990         uint16_t *dst16_ptr;
3991         uint32_t *src32_ptr;
3992         uint64_t r0, r1;
3993
3994         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3995
3996         /* Structs. */
3997         dst_struct = t->structs[ip->alu.dst.struct_id];
3998         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3999
4000         src_struct = t->structs[ip->alu.src.struct_id];
4001         src32_ptr = (uint32_t *)&src_struct[0];
4002
4003         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4004         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4005         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4006         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4007         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4008
4009         /* The first input is a 16-bit number. The second input is a 19-bit
4010          * number. Their sum is a 20-bit number.
4011          */
4012         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4013
4014         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4015          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4016          */
4017         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4018
4019         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4020          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4021          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4022          * generated, therefore the output r is always a 16-bit number.
4023          */
4024         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4025
4026         r0 = ~r0 & 0xFFFF;
4027         r0 = r0 ? r0 : 0xFFFF;
4028
4029         *dst16_ptr = (uint16_t)r0;
4030
4031         /* Thread. */
4032         thread_ip_inc(p);
4033 }
4034
4035 static inline void
4036 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4037 {
4038         struct thread *t = &p->threads[p->thread_id];
4039         struct instruction *ip = t->ip;
4040         uint8_t *dst_struct, *src_struct;
4041         uint16_t *dst16_ptr;
4042         uint32_t *src32_ptr;
4043         uint64_t r = 0;
4044         uint32_t i;
4045
4046         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4047
4048         /* Structs. */
4049         dst_struct = t->structs[ip->alu.dst.struct_id];
4050         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4051
4052         src_struct = t->structs[ip->alu.src.struct_id];
4053         src32_ptr = (uint32_t *)&src_struct[0];
4054
4055         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4056          * Therefore, in the worst case scenario, a 35-bit number is added to a
4057          * 16-bit number (the input r), so the output r is 36-bit number.
4058          */
4059         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4060                 r += *src32_ptr;
4061
4062         /* The first input is a 16-bit number. The second input is a 20-bit
4063          * number. Their sum is a 21-bit number.
4064          */
4065         r = (r & 0xFFFF) + (r >> 16);
4066
4067         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4068          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4069          */
4070         r = (r & 0xFFFF) + (r >> 16);
4071
4072         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4073          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4074          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4075          * generated, therefore the output r is always a 16-bit number.
4076          */
4077         r = (r & 0xFFFF) + (r >> 16);
4078
4079         r = ~r & 0xFFFF;
4080         r = r ? r : 0xFFFF;
4081
4082         *dst16_ptr = (uint16_t)r;
4083
4084         /* Thread. */
4085         thread_ip_inc(p);
4086 }
4087
4088 /*
4089  * Register array.
4090  */
4091 static struct regarray *
4092 regarray_find(struct rte_swx_pipeline *p, const char *name);
4093
4094 static int
4095 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4096                       struct action *action,
4097                       char **tokens,
4098                       int n_tokens,
4099                       struct instruction *instr,
4100                       struct instruction_data *data __rte_unused)
4101 {
4102         char *regarray = tokens[1], *idx = tokens[2];
4103         struct regarray *r;
4104         struct field *fidx;
4105         uint32_t idx_struct_id, idx_val;
4106
4107         CHECK(n_tokens == 3, EINVAL);
4108
4109         r = regarray_find(p, regarray);
4110         CHECK(r, EINVAL);
4111
4112         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4113         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4114         if (fidx) {
4115                 CHECK(!fidx->var_size, EINVAL);
4116
4117                 instr->type = INSTR_REGPREFETCH_RM;
4118                 if (idx[0] == 'h')
4119                         instr->type = INSTR_REGPREFETCH_RH;
4120
4121                 instr->regarray.regarray_id = r->id;
4122                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4123                 instr->regarray.idx.n_bits = fidx->n_bits;
4124                 instr->regarray.idx.offset = fidx->offset / 8;
4125                 instr->regarray.dstsrc_val = 0; /* Unused. */
4126                 return 0;
4127         }
4128
4129         /* REGPREFETCH_RI. */
4130         idx_val = strtoul(idx, &idx, 0);
4131         CHECK(!idx[0], EINVAL);
4132
4133         instr->type = INSTR_REGPREFETCH_RI;
4134         instr->regarray.regarray_id = r->id;
4135         instr->regarray.idx_val = idx_val;
4136         instr->regarray.dstsrc_val = 0; /* Unused. */
4137         return 0;
4138 }
4139
4140 static int
4141 instr_regrd_translate(struct rte_swx_pipeline *p,
4142                       struct action *action,
4143                       char **tokens,
4144                       int n_tokens,
4145                       struct instruction *instr,
4146                       struct instruction_data *data __rte_unused)
4147 {
4148         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4149         struct regarray *r;
4150         struct field *fdst, *fidx;
4151         uint32_t dst_struct_id, idx_struct_id, idx_val;
4152
4153         CHECK(n_tokens == 4, EINVAL);
4154
4155         r = regarray_find(p, regarray);
4156         CHECK(r, EINVAL);
4157
4158         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4159         CHECK(fdst, EINVAL);
4160         CHECK(!fdst->var_size, EINVAL);
4161
4162         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4163         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4164         if (fidx) {
4165                 CHECK(!fidx->var_size, EINVAL);
4166
4167                 instr->type = INSTR_REGRD_MRM;
4168                 if (dst[0] == 'h' && idx[0] != 'h')
4169                         instr->type = INSTR_REGRD_HRM;
4170                 if (dst[0] != 'h' && idx[0] == 'h')
4171                         instr->type = INSTR_REGRD_MRH;
4172                 if (dst[0] == 'h' && idx[0] == 'h')
4173                         instr->type = INSTR_REGRD_HRH;
4174
4175                 instr->regarray.regarray_id = r->id;
4176                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4177                 instr->regarray.idx.n_bits = fidx->n_bits;
4178                 instr->regarray.idx.offset = fidx->offset / 8;
4179                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4180                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4181                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4182                 return 0;
4183         }
4184
4185         /* REGRD_MRI, REGRD_HRI. */
4186         idx_val = strtoul(idx, &idx, 0);
4187         CHECK(!idx[0], EINVAL);
4188
4189         instr->type = INSTR_REGRD_MRI;
4190         if (dst[0] == 'h')
4191                 instr->type = INSTR_REGRD_HRI;
4192
4193         instr->regarray.regarray_id = r->id;
4194         instr->regarray.idx_val = idx_val;
4195         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4196         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4197         instr->regarray.dstsrc.offset = fdst->offset / 8;
4198         return 0;
4199 }
4200
4201 static int
4202 instr_regwr_translate(struct rte_swx_pipeline *p,
4203                       struct action *action,
4204                       char **tokens,
4205                       int n_tokens,
4206                       struct instruction *instr,
4207                       struct instruction_data *data __rte_unused)
4208 {
4209         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4210         struct regarray *r;
4211         struct field *fidx, *fsrc;
4212         uint64_t src_val;
4213         uint32_t idx_struct_id, idx_val, src_struct_id;
4214
4215         CHECK(n_tokens == 4, EINVAL);
4216
4217         r = regarray_find(p, regarray);
4218         CHECK(r, EINVAL);
4219
4220         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4221         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4222         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4223         if (fidx && fsrc) {
4224                 CHECK(!fidx->var_size, EINVAL);
4225                 CHECK(!fsrc->var_size, EINVAL);
4226
4227                 instr->type = INSTR_REGWR_RMM;
4228                 if (idx[0] == 'h' && src[0] != 'h')
4229                         instr->type = INSTR_REGWR_RHM;
4230                 if (idx[0] != 'h' && src[0] == 'h')
4231                         instr->type = INSTR_REGWR_RMH;
4232                 if (idx[0] == 'h' && src[0] == 'h')
4233                         instr->type = INSTR_REGWR_RHH;
4234
4235                 instr->regarray.regarray_id = r->id;
4236                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4237                 instr->regarray.idx.n_bits = fidx->n_bits;
4238                 instr->regarray.idx.offset = fidx->offset / 8;
4239                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4240                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4241                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4242                 return 0;
4243         }
4244
4245         /* REGWR_RHI, REGWR_RMI. */
4246         if (fidx && !fsrc) {
4247                 CHECK(!fidx->var_size, EINVAL);
4248
4249                 src_val = strtoull(src, &src, 0);
4250                 CHECK(!src[0], EINVAL);
4251
4252                 instr->type = INSTR_REGWR_RMI;
4253                 if (idx[0] == 'h')
4254                         instr->type = INSTR_REGWR_RHI;
4255
4256                 instr->regarray.regarray_id = r->id;
4257                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4258                 instr->regarray.idx.n_bits = fidx->n_bits;
4259                 instr->regarray.idx.offset = fidx->offset / 8;
4260                 instr->regarray.dstsrc_val = src_val;
4261                 return 0;
4262         }
4263
4264         /* REGWR_RIH, REGWR_RIM. */
4265         if (!fidx && fsrc) {
4266                 idx_val = strtoul(idx, &idx, 0);
4267                 CHECK(!idx[0], EINVAL);
4268
4269                 CHECK(!fsrc->var_size, EINVAL);
4270
4271                 instr->type = INSTR_REGWR_RIM;
4272                 if (src[0] == 'h')
4273                         instr->type = INSTR_REGWR_RIH;
4274
4275                 instr->regarray.regarray_id = r->id;
4276                 instr->regarray.idx_val = idx_val;
4277                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4278                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4279                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4280                 return 0;
4281         }
4282
4283         /* REGWR_RII. */
4284         src_val = strtoull(src, &src, 0);
4285         CHECK(!src[0], EINVAL);
4286
4287         idx_val = strtoul(idx, &idx, 0);
4288         CHECK(!idx[0], EINVAL);
4289
4290         instr->type = INSTR_REGWR_RII;
4291         instr->regarray.idx_val = idx_val;
4292         instr->regarray.dstsrc_val = src_val;
4293
4294         return 0;
4295 }
4296
4297 static int
4298 instr_regadd_translate(struct rte_swx_pipeline *p,
4299                        struct action *action,
4300                        char **tokens,
4301                        int n_tokens,
4302                        struct instruction *instr,
4303                        struct instruction_data *data __rte_unused)
4304 {
4305         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4306         struct regarray *r;
4307         struct field *fidx, *fsrc;
4308         uint64_t src_val;
4309         uint32_t idx_struct_id, idx_val, src_struct_id;
4310
4311         CHECK(n_tokens == 4, EINVAL);
4312
4313         r = regarray_find(p, regarray);
4314         CHECK(r, EINVAL);
4315
4316         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4317         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4318         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4319         if (fidx && fsrc) {
4320                 CHECK(!fidx->var_size, EINVAL);
4321                 CHECK(!fsrc->var_size, EINVAL);
4322
4323                 instr->type = INSTR_REGADD_RMM;
4324                 if (idx[0] == 'h' && src[0] != 'h')
4325                         instr->type = INSTR_REGADD_RHM;
4326                 if (idx[0] != 'h' && src[0] == 'h')
4327                         instr->type = INSTR_REGADD_RMH;
4328                 if (idx[0] == 'h' && src[0] == 'h')
4329                         instr->type = INSTR_REGADD_RHH;
4330
4331                 instr->regarray.regarray_id = r->id;
4332                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4333                 instr->regarray.idx.n_bits = fidx->n_bits;
4334                 instr->regarray.idx.offset = fidx->offset / 8;
4335                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4336                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4337                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4338                 return 0;
4339         }
4340
4341         /* REGADD_RHI, REGADD_RMI. */
4342         if (fidx && !fsrc) {
4343                 CHECK(!fidx->var_size, EINVAL);
4344
4345                 src_val = strtoull(src, &src, 0);
4346                 CHECK(!src[0], EINVAL);
4347
4348                 instr->type = INSTR_REGADD_RMI;
4349                 if (idx[0] == 'h')
4350                         instr->type = INSTR_REGADD_RHI;
4351
4352                 instr->regarray.regarray_id = r->id;
4353                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4354                 instr->regarray.idx.n_bits = fidx->n_bits;
4355                 instr->regarray.idx.offset = fidx->offset / 8;
4356                 instr->regarray.dstsrc_val = src_val;
4357                 return 0;
4358         }
4359
4360         /* REGADD_RIH, REGADD_RIM. */
4361         if (!fidx && fsrc) {
4362                 idx_val = strtoul(idx, &idx, 0);
4363                 CHECK(!idx[0], EINVAL);
4364
4365                 CHECK(!fsrc->var_size, EINVAL);
4366
4367                 instr->type = INSTR_REGADD_RIM;
4368                 if (src[0] == 'h')
4369                         instr->type = INSTR_REGADD_RIH;
4370
4371                 instr->regarray.regarray_id = r->id;
4372                 instr->regarray.idx_val = idx_val;
4373                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4374                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4375                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4376                 return 0;
4377         }
4378
4379         /* REGADD_RII. */
4380         src_val = strtoull(src, &src, 0);
4381         CHECK(!src[0], EINVAL);
4382
4383         idx_val = strtoul(idx, &idx, 0);
4384         CHECK(!idx[0], EINVAL);
4385
4386         instr->type = INSTR_REGADD_RII;
4387         instr->regarray.idx_val = idx_val;
4388         instr->regarray.dstsrc_val = src_val;
4389         return 0;
4390 }
4391
4392 static inline uint64_t *
4393 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4394 {
4395         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4396         return r->regarray;
4397 }
4398
4399 static inline uint64_t
4400 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4401 {
4402         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4403
4404         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4405         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4406         uint64_t idx64 = *idx64_ptr;
4407         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4408         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4409
4410         return idx;
4411 }
4412
4413 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4414
4415 static inline uint64_t
4416 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4417 {
4418         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4419
4420         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4421         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4422         uint64_t idx64 = *idx64_ptr;
4423         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4424
4425         return idx;
4426 }
4427
4428 #else
4429
4430 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4431
4432 #endif
4433
4434 static inline uint64_t
4435 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4436 {
4437         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4438
4439         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4440
4441         return idx;
4442 }
4443
4444 static inline uint64_t
4445 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4446 {
4447         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4448         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4449         uint64_t src64 = *src64_ptr;
4450         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4451         uint64_t src = src64 & src64_mask;
4452
4453         return src;
4454 }
4455
4456 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4457
4458 static inline uint64_t
4459 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4460 {
4461         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4462         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4463         uint64_t src64 = *src64_ptr;
4464         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4465
4466         return src;
4467 }
4468
4469 #else
4470
4471 #define instr_regarray_src_nbo instr_regarray_src_hbo
4472
4473 #endif
4474
4475 static inline void
4476 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4477 {
4478         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4479         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4480         uint64_t dst64 = *dst64_ptr;
4481         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4482
4483         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4484
4485 }
4486
4487 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4488
4489 static inline void
4490 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4491 {
4492         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4493         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4494         uint64_t dst64 = *dst64_ptr;
4495         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4496
4497         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4498         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4499 }
4500
4501 #else
4502
4503 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4504
4505 #endif
4506
4507 static inline void
4508 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4509 {
4510         struct thread *t = &p->threads[p->thread_id];
4511         struct instruction *ip = t->ip;
4512         uint64_t *regarray, idx;
4513
4514         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4515
4516         /* Structs. */
4517         regarray = instr_regarray_regarray(p, ip);
4518         idx = instr_regarray_idx_nbo(p, t, ip);
4519         rte_prefetch0(&regarray[idx]);
4520
4521         /* Thread. */
4522         thread_ip_inc(p);
4523 }
4524
4525 static inline void
4526 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4527 {
4528         struct thread *t = &p->threads[p->thread_id];
4529         struct instruction *ip = t->ip;
4530         uint64_t *regarray, idx;
4531
4532         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4533
4534         /* Structs. */
4535         regarray = instr_regarray_regarray(p, ip);
4536         idx = instr_regarray_idx_hbo(p, t, ip);
4537         rte_prefetch0(&regarray[idx]);
4538
4539         /* Thread. */
4540         thread_ip_inc(p);
4541 }
4542
4543 static inline void
4544 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4545 {
4546         struct thread *t = &p->threads[p->thread_id];
4547         struct instruction *ip = t->ip;
4548         uint64_t *regarray, idx;
4549
4550         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4551
4552         /* Structs. */
4553         regarray = instr_regarray_regarray(p, ip);
4554         idx = instr_regarray_idx_imm(p, ip);
4555         rte_prefetch0(&regarray[idx]);
4556
4557         /* Thread. */
4558         thread_ip_inc(p);
4559 }
4560
4561 static inline void
4562 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4563 {
4564         struct thread *t = &p->threads[p->thread_id];
4565         struct instruction *ip = t->ip;
4566         uint64_t *regarray, idx;
4567
4568         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4569
4570         /* Structs. */
4571         regarray = instr_regarray_regarray(p, ip);
4572         idx = instr_regarray_idx_nbo(p, t, ip);
4573         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4574
4575         /* Thread. */
4576         thread_ip_inc(p);
4577 }
4578
4579 static inline void
4580 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4581 {
4582         struct thread *t = &p->threads[p->thread_id];
4583         struct instruction *ip = t->ip;
4584         uint64_t *regarray, idx;
4585
4586         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4587
4588         /* Structs. */
4589         regarray = instr_regarray_regarray(p, ip);
4590         idx = instr_regarray_idx_hbo(p, t, ip);
4591         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4592
4593         /* Thread. */
4594         thread_ip_inc(p);
4595 }
4596
4597 static inline void
4598 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4599 {
4600         struct thread *t = &p->threads[p->thread_id];
4601         struct instruction *ip = t->ip;
4602         uint64_t *regarray, idx;
4603
4604         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4605
4606         /* Structs. */
4607         regarray = instr_regarray_regarray(p, ip);
4608         idx = instr_regarray_idx_nbo(p, t, ip);
4609         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4610
4611         /* Thread. */
4612         thread_ip_inc(p);
4613 }
4614
4615 static inline void
4616 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4617 {
4618         struct thread *t = &p->threads[p->thread_id];
4619         struct instruction *ip = t->ip;
4620         uint64_t *regarray, idx;
4621
4622         /* Structs. */
4623         regarray = instr_regarray_regarray(p, ip);
4624         idx = instr_regarray_idx_hbo(p, t, ip);
4625         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4626
4627         /* Thread. */
4628         thread_ip_inc(p);
4629 }
4630
4631 static inline void
4632 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4633 {
4634         struct thread *t = &p->threads[p->thread_id];
4635         struct instruction *ip = t->ip;
4636         uint64_t *regarray, idx;
4637
4638         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4639
4640         /* Structs. */
4641         regarray = instr_regarray_regarray(p, ip);
4642         idx = instr_regarray_idx_imm(p, ip);
4643         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4644
4645         /* Thread. */
4646         thread_ip_inc(p);
4647 }
4648
4649 static inline void
4650 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4651 {
4652         struct thread *t = &p->threads[p->thread_id];
4653         struct instruction *ip = t->ip;
4654         uint64_t *regarray, idx;
4655
4656         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4657
4658         /* Structs. */
4659         regarray = instr_regarray_regarray(p, ip);
4660         idx = instr_regarray_idx_imm(p, ip);
4661         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4662
4663         /* Thread. */
4664         thread_ip_inc(p);
4665 }
4666
4667 static inline void
4668 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4669 {
4670         struct thread *t = &p->threads[p->thread_id];
4671         struct instruction *ip = t->ip;
4672         uint64_t *regarray, idx, src;
4673
4674         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4675
4676         /* Structs. */
4677         regarray = instr_regarray_regarray(p, ip);
4678         idx = instr_regarray_idx_nbo(p, t, ip);
4679         src = instr_regarray_src_nbo(t, ip);
4680         regarray[idx] = src;
4681
4682         /* Thread. */
4683         thread_ip_inc(p);
4684 }
4685
4686 static inline void
4687 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4688 {
4689         struct thread *t = &p->threads[p->thread_id];
4690         struct instruction *ip = t->ip;
4691         uint64_t *regarray, idx, src;
4692
4693         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4694
4695         /* Structs. */
4696         regarray = instr_regarray_regarray(p, ip);
4697         idx = instr_regarray_idx_nbo(p, t, ip);
4698         src = instr_regarray_src_hbo(t, ip);
4699         regarray[idx] = src;
4700
4701         /* Thread. */
4702         thread_ip_inc(p);
4703 }
4704
4705 static inline void
4706 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4707 {
4708         struct thread *t = &p->threads[p->thread_id];
4709         struct instruction *ip = t->ip;
4710         uint64_t *regarray, idx, src;
4711
4712         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4713
4714         /* Structs. */
4715         regarray = instr_regarray_regarray(p, ip);
4716         idx = instr_regarray_idx_hbo(p, t, ip);
4717         src = instr_regarray_src_nbo(t, ip);
4718         regarray[idx] = src;
4719
4720         /* Thread. */
4721         thread_ip_inc(p);
4722 }
4723
4724 static inline void
4725 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4726 {
4727         struct thread *t = &p->threads[p->thread_id];
4728         struct instruction *ip = t->ip;
4729         uint64_t *regarray, idx, src;
4730
4731         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4732
4733         /* Structs. */
4734         regarray = instr_regarray_regarray(p, ip);
4735         idx = instr_regarray_idx_hbo(p, t, ip);
4736         src = instr_regarray_src_hbo(t, ip);
4737         regarray[idx] = src;
4738
4739         /* Thread. */
4740         thread_ip_inc(p);
4741 }
4742
4743 static inline void
4744 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4745 {
4746         struct thread *t = &p->threads[p->thread_id];
4747         struct instruction *ip = t->ip;
4748         uint64_t *regarray, idx, src;
4749
4750         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4751
4752         /* Structs. */
4753         regarray = instr_regarray_regarray(p, ip);
4754         idx = instr_regarray_idx_nbo(p, t, ip);
4755         src = ip->regarray.dstsrc_val;
4756         regarray[idx] = src;
4757
4758         /* Thread. */
4759         thread_ip_inc(p);
4760 }
4761
4762 static inline void
4763 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4764 {
4765         struct thread *t = &p->threads[p->thread_id];
4766         struct instruction *ip = t->ip;
4767         uint64_t *regarray, idx, src;
4768
4769         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4770
4771         /* Structs. */
4772         regarray = instr_regarray_regarray(p, ip);
4773         idx = instr_regarray_idx_hbo(p, t, ip);
4774         src = ip->regarray.dstsrc_val;
4775         regarray[idx] = src;
4776
4777         /* Thread. */
4778         thread_ip_inc(p);
4779 }
4780
4781 static inline void
4782 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4783 {
4784         struct thread *t = &p->threads[p->thread_id];
4785         struct instruction *ip = t->ip;
4786         uint64_t *regarray, idx, src;
4787
4788         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4789
4790         /* Structs. */
4791         regarray = instr_regarray_regarray(p, ip);
4792         idx = instr_regarray_idx_imm(p, ip);
4793         src = instr_regarray_src_nbo(t, ip);
4794         regarray[idx] = src;
4795
4796         /* Thread. */
4797         thread_ip_inc(p);
4798 }
4799
4800 static inline void
4801 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4802 {
4803         struct thread *t = &p->threads[p->thread_id];
4804         struct instruction *ip = t->ip;
4805         uint64_t *regarray, idx, src;
4806
4807         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4808
4809         /* Structs. */
4810         regarray = instr_regarray_regarray(p, ip);
4811         idx = instr_regarray_idx_imm(p, ip);
4812         src = instr_regarray_src_hbo(t, ip);
4813         regarray[idx] = src;
4814
4815         /* Thread. */
4816         thread_ip_inc(p);
4817 }
4818
4819 static inline void
4820 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4821 {
4822         struct thread *t = &p->threads[p->thread_id];
4823         struct instruction *ip = t->ip;
4824         uint64_t *regarray, idx, src;
4825
4826         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4827
4828         /* Structs. */
4829         regarray = instr_regarray_regarray(p, ip);
4830         idx = instr_regarray_idx_imm(p, ip);
4831         src = ip->regarray.dstsrc_val;
4832         regarray[idx] = src;
4833
4834         /* Thread. */
4835         thread_ip_inc(p);
4836 }
4837
4838 static inline void
4839 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4840 {
4841         struct thread *t = &p->threads[p->thread_id];
4842         struct instruction *ip = t->ip;
4843         uint64_t *regarray, idx, src;
4844
4845         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4846
4847         /* Structs. */
4848         regarray = instr_regarray_regarray(p, ip);
4849         idx = instr_regarray_idx_nbo(p, t, ip);
4850         src = instr_regarray_src_nbo(t, ip);
4851         regarray[idx] += src;
4852
4853         /* Thread. */
4854         thread_ip_inc(p);
4855 }
4856
4857 static inline void
4858 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4859 {
4860         struct thread *t = &p->threads[p->thread_id];
4861         struct instruction *ip = t->ip;
4862         uint64_t *regarray, idx, src;
4863
4864         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4865
4866         /* Structs. */
4867         regarray = instr_regarray_regarray(p, ip);
4868         idx = instr_regarray_idx_nbo(p, t, ip);
4869         src = instr_regarray_src_hbo(t, ip);
4870         regarray[idx] += src;
4871
4872         /* Thread. */
4873         thread_ip_inc(p);
4874 }
4875
4876 static inline void
4877 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4878 {
4879         struct thread *t = &p->threads[p->thread_id];
4880         struct instruction *ip = t->ip;
4881         uint64_t *regarray, idx, src;
4882
4883         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4884
4885         /* Structs. */
4886         regarray = instr_regarray_regarray(p, ip);
4887         idx = instr_regarray_idx_hbo(p, t, ip);
4888         src = instr_regarray_src_nbo(t, ip);
4889         regarray[idx] += src;
4890
4891         /* Thread. */
4892         thread_ip_inc(p);
4893 }
4894
4895 static inline void
4896 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4897 {
4898         struct thread *t = &p->threads[p->thread_id];
4899         struct instruction *ip = t->ip;
4900         uint64_t *regarray, idx, src;
4901
4902         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4903
4904         /* Structs. */
4905         regarray = instr_regarray_regarray(p, ip);
4906         idx = instr_regarray_idx_hbo(p, t, ip);
4907         src = instr_regarray_src_hbo(t, ip);
4908         regarray[idx] += src;
4909
4910         /* Thread. */
4911         thread_ip_inc(p);
4912 }
4913
4914 static inline void
4915 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4916 {
4917         struct thread *t = &p->threads[p->thread_id];
4918         struct instruction *ip = t->ip;
4919         uint64_t *regarray, idx, src;
4920
4921         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4922
4923         /* Structs. */
4924         regarray = instr_regarray_regarray(p, ip);
4925         idx = instr_regarray_idx_nbo(p, t, ip);
4926         src = ip->regarray.dstsrc_val;
4927         regarray[idx] += src;
4928
4929         /* Thread. */
4930         thread_ip_inc(p);
4931 }
4932
4933 static inline void
4934 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4935 {
4936         struct thread *t = &p->threads[p->thread_id];
4937         struct instruction *ip = t->ip;
4938         uint64_t *regarray, idx, src;
4939
4940         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4941
4942         /* Structs. */
4943         regarray = instr_regarray_regarray(p, ip);
4944         idx = instr_regarray_idx_hbo(p, t, ip);
4945         src = ip->regarray.dstsrc_val;
4946         regarray[idx] += src;
4947
4948         /* Thread. */
4949         thread_ip_inc(p);
4950 }
4951
4952 static inline void
4953 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4954 {
4955         struct thread *t = &p->threads[p->thread_id];
4956         struct instruction *ip = t->ip;
4957         uint64_t *regarray, idx, src;
4958
4959         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4960
4961         /* Structs. */
4962         regarray = instr_regarray_regarray(p, ip);
4963         idx = instr_regarray_idx_imm(p, ip);
4964         src = instr_regarray_src_nbo(t, ip);
4965         regarray[idx] += src;
4966
4967         /* Thread. */
4968         thread_ip_inc(p);
4969 }
4970
4971 static inline void
4972 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4973 {
4974         struct thread *t = &p->threads[p->thread_id];
4975         struct instruction *ip = t->ip;
4976         uint64_t *regarray, idx, src;
4977
4978         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4979
4980         /* Structs. */
4981         regarray = instr_regarray_regarray(p, ip);
4982         idx = instr_regarray_idx_imm(p, ip);
4983         src = instr_regarray_src_hbo(t, ip);
4984         regarray[idx] += src;
4985
4986         /* Thread. */
4987         thread_ip_inc(p);
4988 }
4989
4990 static inline void
4991 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4992 {
4993         struct thread *t = &p->threads[p->thread_id];
4994         struct instruction *ip = t->ip;
4995         uint64_t *regarray, idx, src;
4996
4997         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4998
4999         /* Structs. */
5000         regarray = instr_regarray_regarray(p, ip);
5001         idx = instr_regarray_idx_imm(p, ip);
5002         src = ip->regarray.dstsrc_val;
5003         regarray[idx] += src;
5004
5005         /* Thread. */
5006         thread_ip_inc(p);
5007 }
5008
5009 /*
5010  * metarray.
5011  */
5012 static struct metarray *
5013 metarray_find(struct rte_swx_pipeline *p, const char *name);
5014
5015 static int
5016 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5017                             struct action *action,
5018                             char **tokens,
5019                             int n_tokens,
5020                             struct instruction *instr,
5021                             struct instruction_data *data __rte_unused)
5022 {
5023         char *metarray = tokens[1], *idx = tokens[2];
5024         struct metarray *m;
5025         struct field *fidx;
5026         uint32_t idx_struct_id, idx_val;
5027
5028         CHECK(n_tokens == 3, EINVAL);
5029
5030         m = metarray_find(p, metarray);
5031         CHECK(m, EINVAL);
5032
5033         /* METPREFETCH_H, METPREFETCH_M. */
5034         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5035         if (fidx) {
5036                 CHECK(!fidx->var_size, EINVAL);
5037
5038                 instr->type = INSTR_METPREFETCH_M;
5039                 if (idx[0] == 'h')
5040                         instr->type = INSTR_METPREFETCH_H;
5041
5042                 instr->meter.metarray_id = m->id;
5043                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5044                 instr->meter.idx.n_bits = fidx->n_bits;
5045                 instr->meter.idx.offset = fidx->offset / 8;
5046                 return 0;
5047         }
5048
5049         /* METPREFETCH_I. */
5050         idx_val = strtoul(idx, &idx, 0);
5051         CHECK(!idx[0], EINVAL);
5052
5053         instr->type = INSTR_METPREFETCH_I;
5054         instr->meter.metarray_id = m->id;
5055         instr->meter.idx_val = idx_val;
5056         return 0;
5057 }
5058
5059 static int
5060 instr_meter_translate(struct rte_swx_pipeline *p,
5061                       struct action *action,
5062                       char **tokens,
5063                       int n_tokens,
5064                       struct instruction *instr,
5065                       struct instruction_data *data __rte_unused)
5066 {
5067         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5068         char *color_in = tokens[4], *color_out = tokens[5];
5069         struct metarray *m;
5070         struct field *fidx, *flength, *fcin, *fcout;
5071         uint32_t idx_struct_id, length_struct_id;
5072         uint32_t color_in_struct_id, color_out_struct_id;
5073
5074         CHECK(n_tokens == 6, EINVAL);
5075
5076         m = metarray_find(p, metarray);
5077         CHECK(m, EINVAL);
5078
5079         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5080
5081         flength = struct_field_parse(p, action, length, &length_struct_id);
5082         CHECK(flength, EINVAL);
5083         CHECK(!flength->var_size, EINVAL);
5084
5085         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5086
5087         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5088         CHECK(fcout, EINVAL);
5089         CHECK(!fcout->var_size, EINVAL);
5090
5091         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5092         if (fidx && fcin) {
5093                 CHECK(!fidx->var_size, EINVAL);
5094                 CHECK(!fcin->var_size, EINVAL);
5095
5096                 instr->type = INSTR_METER_MMM;
5097                 if (idx[0] == 'h' && length[0] == 'h')
5098                         instr->type = INSTR_METER_HHM;
5099                 if (idx[0] == 'h' && length[0] != 'h')
5100                         instr->type = INSTR_METER_HMM;
5101                 if (idx[0] != 'h' && length[0] == 'h')
5102                         instr->type = INSTR_METER_MHM;
5103
5104                 instr->meter.metarray_id = m->id;
5105
5106                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5107                 instr->meter.idx.n_bits = fidx->n_bits;
5108                 instr->meter.idx.offset = fidx->offset / 8;
5109
5110                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5111                 instr->meter.length.n_bits = flength->n_bits;
5112                 instr->meter.length.offset = flength->offset / 8;
5113
5114                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5115                 instr->meter.color_in.n_bits = fcin->n_bits;
5116                 instr->meter.color_in.offset = fcin->offset / 8;
5117
5118                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5119                 instr->meter.color_out.n_bits = fcout->n_bits;
5120                 instr->meter.color_out.offset = fcout->offset / 8;
5121
5122                 return 0;
5123         }
5124
5125         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5126         if (fidx && !fcin) {
5127                 uint32_t color_in_val;
5128
5129                 CHECK(!fidx->var_size, EINVAL);
5130
5131                 color_in_val = strtoul(color_in, &color_in, 0);
5132                 CHECK(!color_in[0], EINVAL);
5133
5134                 instr->type = INSTR_METER_MMI;
5135                 if (idx[0] == 'h' && length[0] == 'h')
5136                         instr->type = INSTR_METER_HHI;
5137                 if (idx[0] == 'h' && length[0] != 'h')
5138                         instr->type = INSTR_METER_HMI;
5139                 if (idx[0] != 'h' && length[0] == 'h')
5140                         instr->type = INSTR_METER_MHI;
5141
5142                 instr->meter.metarray_id = m->id;
5143
5144                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5145                 instr->meter.idx.n_bits = fidx->n_bits;
5146                 instr->meter.idx.offset = fidx->offset / 8;
5147
5148                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5149                 instr->meter.length.n_bits = flength->n_bits;
5150                 instr->meter.length.offset = flength->offset / 8;
5151
5152                 instr->meter.color_in_val = color_in_val;
5153
5154                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5155                 instr->meter.color_out.n_bits = fcout->n_bits;
5156                 instr->meter.color_out.offset = fcout->offset / 8;
5157
5158                 return 0;
5159         }
5160
5161         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5162         if (!fidx && fcin) {
5163                 uint32_t idx_val;
5164
5165                 idx_val = strtoul(idx, &idx, 0);
5166                 CHECK(!idx[0], EINVAL);
5167
5168                 CHECK(!fcin->var_size, EINVAL);
5169
5170                 instr->type = INSTR_METER_IMM;
5171                 if (length[0] == 'h')
5172                         instr->type = INSTR_METER_IHM;
5173
5174                 instr->meter.metarray_id = m->id;
5175
5176                 instr->meter.idx_val = idx_val;
5177
5178                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5179                 instr->meter.length.n_bits = flength->n_bits;
5180                 instr->meter.length.offset = flength->offset / 8;
5181
5182                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5183                 instr->meter.color_in.n_bits = fcin->n_bits;
5184                 instr->meter.color_in.offset = fcin->offset / 8;
5185
5186                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5187                 instr->meter.color_out.n_bits = fcout->n_bits;
5188                 instr->meter.color_out.offset = fcout->offset / 8;
5189
5190                 return 0;
5191         }
5192
5193         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5194         if (!fidx && !fcin) {
5195                 uint32_t idx_val, color_in_val;
5196
5197                 idx_val = strtoul(idx, &idx, 0);
5198                 CHECK(!idx[0], EINVAL);
5199
5200                 color_in_val = strtoul(color_in, &color_in, 0);
5201                 CHECK(!color_in[0], EINVAL);
5202
5203                 instr->type = INSTR_METER_IMI;
5204                 if (length[0] == 'h')
5205                         instr->type = INSTR_METER_IHI;
5206
5207                 instr->meter.metarray_id = m->id;
5208
5209                 instr->meter.idx_val = idx_val;
5210
5211                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5212                 instr->meter.length.n_bits = flength->n_bits;
5213                 instr->meter.length.offset = flength->offset / 8;
5214
5215                 instr->meter.color_in_val = color_in_val;
5216
5217                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5218                 instr->meter.color_out.n_bits = fcout->n_bits;
5219                 instr->meter.color_out.offset = fcout->offset / 8;
5220
5221                 return 0;
5222         }
5223
5224         CHECK(0, EINVAL);
5225 }
5226
5227 static inline struct meter *
5228 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5229 {
5230         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5231
5232         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5233         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5234         uint64_t idx64 = *idx64_ptr;
5235         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5236         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5237
5238         return &r->metarray[idx];
5239 }
5240
5241 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5242
5243 static inline struct meter *
5244 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5245 {
5246         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5247
5248         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5249         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5250         uint64_t idx64 = *idx64_ptr;
5251         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5252
5253         return &r->metarray[idx];
5254 }
5255
5256 #else
5257
5258 #define instr_meter_idx_nbo instr_meter_idx_hbo
5259
5260 #endif
5261
5262 static inline struct meter *
5263 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5264 {
5265         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5266
5267         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5268
5269         return &r->metarray[idx];
5270 }
5271
5272 static inline uint32_t
5273 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5274 {
5275         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5276         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5277         uint64_t src64 = *src64_ptr;
5278         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5279         uint64_t src = src64 & src64_mask;
5280
5281         return (uint32_t)src;
5282 }
5283
5284 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5285
5286 static inline uint32_t
5287 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5288 {
5289         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5290         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5291         uint64_t src64 = *src64_ptr;
5292         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5293
5294         return (uint32_t)src;
5295 }
5296
5297 #else
5298
5299 #define instr_meter_length_nbo instr_meter_length_hbo
5300
5301 #endif
5302
5303 static inline enum rte_color
5304 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5305 {
5306         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5307         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5308         uint64_t src64 = *src64_ptr;
5309         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5310         uint64_t src = src64 & src64_mask;
5311
5312         return (enum rte_color)src;
5313 }
5314
5315 static inline void
5316 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5317 {
5318         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5319         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5320         uint64_t dst64 = *dst64_ptr;
5321         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5322
5323         uint64_t src = (uint64_t)color_out;
5324
5325         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5326 }
5327
5328 static inline void
5329 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5330 {
5331         struct thread *t = &p->threads[p->thread_id];
5332         struct instruction *ip = t->ip;
5333         struct meter *m;
5334
5335         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5336
5337         /* Structs. */
5338         m = instr_meter_idx_nbo(p, t, ip);
5339         rte_prefetch0(m);
5340
5341         /* Thread. */
5342         thread_ip_inc(p);
5343 }
5344
5345 static inline void
5346 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5347 {
5348         struct thread *t = &p->threads[p->thread_id];
5349         struct instruction *ip = t->ip;
5350         struct meter *m;
5351
5352         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5353
5354         /* Structs. */
5355         m = instr_meter_idx_hbo(p, t, ip);
5356         rte_prefetch0(m);
5357
5358         /* Thread. */
5359         thread_ip_inc(p);
5360 }
5361
5362 static inline void
5363 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5364 {
5365         struct thread *t = &p->threads[p->thread_id];
5366         struct instruction *ip = t->ip;
5367         struct meter *m;
5368
5369         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5370
5371         /* Structs. */
5372         m = instr_meter_idx_imm(p, ip);
5373         rte_prefetch0(m);
5374
5375         /* Thread. */
5376         thread_ip_inc(p);
5377 }
5378
5379 static inline void
5380 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5381 {
5382         struct thread *t = &p->threads[p->thread_id];
5383         struct instruction *ip = t->ip;
5384         struct meter *m;
5385         uint64_t time, n_pkts, n_bytes;
5386         uint32_t length;
5387         enum rte_color color_in, color_out;
5388
5389         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5390
5391         /* Structs. */
5392         m = instr_meter_idx_nbo(p, t, ip);
5393         rte_prefetch0(m->n_pkts);
5394         time = rte_get_tsc_cycles();
5395         length = instr_meter_length_nbo(t, ip);
5396         color_in = instr_meter_color_in_hbo(t, ip);
5397
5398         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5399                 &m->profile->profile,
5400                 time,
5401                 length,
5402                 color_in);
5403
5404         color_out &= m->color_mask;
5405
5406         n_pkts = m->n_pkts[color_out];
5407         n_bytes = m->n_bytes[color_out];
5408
5409         instr_meter_color_out_hbo_set(t, ip, color_out);
5410
5411         m->n_pkts[color_out] = n_pkts + 1;
5412         m->n_bytes[color_out] = n_bytes + length;
5413
5414         /* Thread. */
5415         thread_ip_inc(p);
5416 }
5417
5418 static inline void
5419 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5420 {
5421         struct thread *t = &p->threads[p->thread_id];
5422         struct instruction *ip = t->ip;
5423         struct meter *m;
5424         uint64_t time, n_pkts, n_bytes;
5425         uint32_t length;
5426         enum rte_color color_in, color_out;
5427
5428         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5429
5430         /* Structs. */
5431         m = instr_meter_idx_nbo(p, t, ip);
5432         rte_prefetch0(m->n_pkts);
5433         time = rte_get_tsc_cycles();
5434         length = instr_meter_length_nbo(t, ip);
5435         color_in = (enum rte_color)ip->meter.color_in_val;
5436
5437         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5438                 &m->profile->profile,
5439                 time,
5440                 length,
5441                 color_in);
5442
5443         color_out &= m->color_mask;
5444
5445         n_pkts = m->n_pkts[color_out];
5446         n_bytes = m->n_bytes[color_out];
5447
5448         instr_meter_color_out_hbo_set(t, ip, color_out);
5449
5450         m->n_pkts[color_out] = n_pkts + 1;
5451         m->n_bytes[color_out] = n_bytes + length;
5452
5453         /* Thread. */
5454         thread_ip_inc(p);
5455 }
5456
5457 static inline void
5458 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5459 {
5460         struct thread *t = &p->threads[p->thread_id];
5461         struct instruction *ip = t->ip;
5462         struct meter *m;
5463         uint64_t time, n_pkts, n_bytes;
5464         uint32_t length;
5465         enum rte_color color_in, color_out;
5466
5467         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5468
5469         /* Structs. */
5470         m = instr_meter_idx_nbo(p, t, ip);
5471         rte_prefetch0(m->n_pkts);
5472         time = rte_get_tsc_cycles();
5473         length = instr_meter_length_hbo(t, ip);
5474         color_in = instr_meter_color_in_hbo(t, ip);
5475
5476         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5477                 &m->profile->profile,
5478                 time,
5479                 length,
5480                 color_in);
5481
5482         color_out &= m->color_mask;
5483
5484         n_pkts = m->n_pkts[color_out];
5485         n_bytes = m->n_bytes[color_out];
5486
5487         instr_meter_color_out_hbo_set(t, ip, color_out);
5488
5489         m->n_pkts[color_out] = n_pkts + 1;
5490         m->n_bytes[color_out] = n_bytes + length;
5491
5492         /* Thread. */
5493         thread_ip_inc(p);
5494 }
5495 static inline void
5496 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5497 {
5498         struct thread *t = &p->threads[p->thread_id];
5499         struct instruction *ip = t->ip;
5500         struct meter *m;
5501         uint64_t time, n_pkts, n_bytes;
5502         uint32_t length;
5503         enum rte_color color_in, color_out;
5504
5505         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5506
5507         /* Structs. */
5508         m = instr_meter_idx_nbo(p, t, ip);
5509         rte_prefetch0(m->n_pkts);
5510         time = rte_get_tsc_cycles();
5511         length = instr_meter_length_hbo(t, ip);
5512         color_in = (enum rte_color)ip->meter.color_in_val;
5513
5514         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5515                 &m->profile->profile,
5516                 time,
5517                 length,
5518                 color_in);
5519
5520         color_out &= m->color_mask;
5521
5522         n_pkts = m->n_pkts[color_out];
5523         n_bytes = m->n_bytes[color_out];
5524
5525         instr_meter_color_out_hbo_set(t, ip, color_out);
5526
5527         m->n_pkts[color_out] = n_pkts + 1;
5528         m->n_bytes[color_out] = n_bytes + length;
5529
5530         /* Thread. */
5531         thread_ip_inc(p);
5532 }
5533
5534 static inline void
5535 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5536 {
5537         struct thread *t = &p->threads[p->thread_id];
5538         struct instruction *ip = t->ip;
5539         struct meter *m;
5540         uint64_t time, n_pkts, n_bytes;
5541         uint32_t length;
5542         enum rte_color color_in, color_out;
5543
5544         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5545
5546         /* Structs. */
5547         m = instr_meter_idx_hbo(p, t, ip);
5548         rte_prefetch0(m->n_pkts);
5549         time = rte_get_tsc_cycles();
5550         length = instr_meter_length_nbo(t, ip);
5551         color_in = instr_meter_color_in_hbo(t, ip);
5552
5553         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5554                 &m->profile->profile,
5555                 time,
5556                 length,
5557                 color_in);
5558
5559         color_out &= m->color_mask;
5560
5561         n_pkts = m->n_pkts[color_out];
5562         n_bytes = m->n_bytes[color_out];
5563
5564         instr_meter_color_out_hbo_set(t, ip, color_out);
5565
5566         m->n_pkts[color_out] = n_pkts + 1;
5567         m->n_bytes[color_out] = n_bytes + length;
5568
5569         /* Thread. */
5570         thread_ip_inc(p);
5571 }
5572
5573 static inline void
5574 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5575 {
5576         struct thread *t = &p->threads[p->thread_id];
5577         struct instruction *ip = t->ip;
5578         struct meter *m;
5579         uint64_t time, n_pkts, n_bytes;
5580         uint32_t length;
5581         enum rte_color color_in, color_out;
5582
5583         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5584
5585         /* Structs. */
5586         m = instr_meter_idx_hbo(p, t, ip);
5587         rte_prefetch0(m->n_pkts);
5588         time = rte_get_tsc_cycles();
5589         length = instr_meter_length_nbo(t, ip);
5590         color_in = (enum rte_color)ip->meter.color_in_val;
5591
5592         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5593                 &m->profile->profile,
5594                 time,
5595                 length,
5596                 color_in);
5597
5598         color_out &= m->color_mask;
5599
5600         n_pkts = m->n_pkts[color_out];
5601         n_bytes = m->n_bytes[color_out];
5602
5603         instr_meter_color_out_hbo_set(t, ip, color_out);
5604
5605         m->n_pkts[color_out] = n_pkts + 1;
5606         m->n_bytes[color_out] = n_bytes + length;
5607
5608         /* Thread. */
5609         thread_ip_inc(p);
5610 }
5611
5612 static inline void
5613 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5614 {
5615         struct thread *t = &p->threads[p->thread_id];
5616         struct instruction *ip = t->ip;
5617         struct meter *m;
5618         uint64_t time, n_pkts, n_bytes;
5619         uint32_t length;
5620         enum rte_color color_in, color_out;
5621
5622         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5623
5624         /* Structs. */
5625         m = instr_meter_idx_hbo(p, t, ip);
5626         rte_prefetch0(m->n_pkts);
5627         time = rte_get_tsc_cycles();
5628         length = instr_meter_length_hbo(t, ip);
5629         color_in = instr_meter_color_in_hbo(t, ip);
5630
5631         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5632                 &m->profile->profile,
5633                 time,
5634                 length,
5635                 color_in);
5636
5637         color_out &= m->color_mask;
5638
5639         n_pkts = m->n_pkts[color_out];
5640         n_bytes = m->n_bytes[color_out];
5641
5642         instr_meter_color_out_hbo_set(t, ip, color_out);
5643
5644         m->n_pkts[color_out] = n_pkts + 1;
5645         m->n_bytes[color_out] = n_bytes + length;
5646
5647         /* Thread. */
5648         thread_ip_inc(p);
5649 }
5650
5651 static inline void
5652 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5653 {
5654         struct thread *t = &p->threads[p->thread_id];
5655         struct instruction *ip = t->ip;
5656         struct meter *m;
5657         uint64_t time, n_pkts, n_bytes;
5658         uint32_t length;
5659         enum rte_color color_in, color_out;
5660
5661         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5662
5663         /* Structs. */
5664         m = instr_meter_idx_hbo(p, t, ip);
5665         rte_prefetch0(m->n_pkts);
5666         time = rte_get_tsc_cycles();
5667         length = instr_meter_length_hbo(t, ip);
5668         color_in = (enum rte_color)ip->meter.color_in_val;
5669
5670         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5671                 &m->profile->profile,
5672                 time,
5673                 length,
5674                 color_in);
5675
5676         color_out &= m->color_mask;
5677
5678         n_pkts = m->n_pkts[color_out];
5679         n_bytes = m->n_bytes[color_out];
5680
5681         instr_meter_color_out_hbo_set(t, ip, color_out);
5682
5683         m->n_pkts[color_out] = n_pkts + 1;
5684         m->n_bytes[color_out] = n_bytes + length;
5685
5686         /* Thread. */
5687         thread_ip_inc(p);
5688 }
5689
5690 static inline void
5691 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5692 {
5693         struct thread *t = &p->threads[p->thread_id];
5694         struct instruction *ip = t->ip;
5695         struct meter *m;
5696         uint64_t time, n_pkts, n_bytes;
5697         uint32_t length;
5698         enum rte_color color_in, color_out;
5699
5700         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5701
5702         /* Structs. */
5703         m = instr_meter_idx_imm(p, ip);
5704         rte_prefetch0(m->n_pkts);
5705         time = rte_get_tsc_cycles();
5706         length = instr_meter_length_nbo(t, ip);
5707         color_in = instr_meter_color_in_hbo(t, ip);
5708
5709         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5710                 &m->profile->profile,
5711                 time,
5712                 length,
5713                 color_in);
5714
5715         color_out &= m->color_mask;
5716
5717         n_pkts = m->n_pkts[color_out];
5718         n_bytes = m->n_bytes[color_out];
5719
5720         instr_meter_color_out_hbo_set(t, ip, color_out);
5721
5722         m->n_pkts[color_out] = n_pkts + 1;
5723         m->n_bytes[color_out] = n_bytes + length;
5724
5725         /* Thread. */
5726         thread_ip_inc(p);
5727 }
5728
5729 static inline void
5730 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5731 {
5732         struct thread *t = &p->threads[p->thread_id];
5733         struct instruction *ip = t->ip;
5734         struct meter *m;
5735         uint64_t time, n_pkts, n_bytes;
5736         uint32_t length;
5737         enum rte_color color_in, color_out;
5738
5739         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5740
5741         /* Structs. */
5742         m = instr_meter_idx_imm(p, ip);
5743         rte_prefetch0(m->n_pkts);
5744         time = rte_get_tsc_cycles();
5745         length = instr_meter_length_nbo(t, ip);
5746         color_in = (enum rte_color)ip->meter.color_in_val;
5747
5748         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5749                 &m->profile->profile,
5750                 time,
5751                 length,
5752                 color_in);
5753
5754         color_out &= m->color_mask;
5755
5756         n_pkts = m->n_pkts[color_out];
5757         n_bytes = m->n_bytes[color_out];
5758
5759         instr_meter_color_out_hbo_set(t, ip, color_out);
5760
5761         m->n_pkts[color_out] = n_pkts + 1;
5762         m->n_bytes[color_out] = n_bytes + length;
5763
5764         /* Thread. */
5765         thread_ip_inc(p);
5766 }
5767
5768 static inline void
5769 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5770 {
5771         struct thread *t = &p->threads[p->thread_id];
5772         struct instruction *ip = t->ip;
5773         struct meter *m;
5774         uint64_t time, n_pkts, n_bytes;
5775         uint32_t length;
5776         enum rte_color color_in, color_out;
5777
5778         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5779
5780         /* Structs. */
5781         m = instr_meter_idx_imm(p, ip);
5782         rte_prefetch0(m->n_pkts);
5783         time = rte_get_tsc_cycles();
5784         length = instr_meter_length_hbo(t, ip);
5785         color_in = instr_meter_color_in_hbo(t, ip);
5786
5787         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5788                 &m->profile->profile,
5789                 time,
5790                 length,
5791                 color_in);
5792
5793         color_out &= m->color_mask;
5794
5795         n_pkts = m->n_pkts[color_out];
5796         n_bytes = m->n_bytes[color_out];
5797
5798         instr_meter_color_out_hbo_set(t, ip, color_out);
5799
5800         m->n_pkts[color_out] = n_pkts + 1;
5801         m->n_bytes[color_out] = n_bytes + length;
5802
5803         /* Thread. */
5804         thread_ip_inc(p);
5805 }
5806 static inline void
5807 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5808 {
5809         struct thread *t = &p->threads[p->thread_id];
5810         struct instruction *ip = t->ip;
5811         struct meter *m;
5812         uint64_t time, n_pkts, n_bytes;
5813         uint32_t length;
5814         enum rte_color color_in, color_out;
5815
5816         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5817
5818         /* Structs. */
5819         m = instr_meter_idx_imm(p, ip);
5820         rte_prefetch0(m->n_pkts);
5821         time = rte_get_tsc_cycles();
5822         length = instr_meter_length_hbo(t, ip);
5823         color_in = (enum rte_color)ip->meter.color_in_val;
5824
5825         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5826                 &m->profile->profile,
5827                 time,
5828                 length,
5829                 color_in);
5830
5831         color_out &= m->color_mask;
5832
5833         n_pkts = m->n_pkts[color_out];
5834         n_bytes = m->n_bytes[color_out];
5835
5836         instr_meter_color_out_hbo_set(t, ip, color_out);
5837
5838         m->n_pkts[color_out] = n_pkts + 1;
5839         m->n_bytes[color_out] = n_bytes + length;
5840
5841         /* Thread. */
5842         thread_ip_inc(p);
5843 }
5844
5845 /*
5846  * jmp.
5847  */
5848 static int
5849 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5850                     struct action *action __rte_unused,
5851                     char **tokens,
5852                     int n_tokens,
5853                     struct instruction *instr,
5854                     struct instruction_data *data)
5855 {
5856         CHECK(n_tokens == 2, EINVAL);
5857
5858         strcpy(data->jmp_label, tokens[1]);
5859
5860         instr->type = INSTR_JMP;
5861         instr->jmp.ip = NULL; /* Resolved later. */
5862         return 0;
5863 }
5864
5865 static int
5866 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5867                           struct action *action __rte_unused,
5868                           char **tokens,
5869                           int n_tokens,
5870                           struct instruction *instr,
5871                           struct instruction_data *data)
5872 {
5873         struct header *h;
5874
5875         CHECK(n_tokens == 3, EINVAL);
5876
5877         strcpy(data->jmp_label, tokens[1]);
5878
5879         h = header_parse(p, tokens[2]);
5880         CHECK(h, EINVAL);
5881
5882         instr->type = INSTR_JMP_VALID;
5883         instr->jmp.ip = NULL; /* Resolved later. */
5884         instr->jmp.header_id = h->id;
5885         return 0;
5886 }
5887
5888 static int
5889 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5890                             struct action *action __rte_unused,
5891                             char **tokens,
5892                             int n_tokens,
5893                             struct instruction *instr,
5894                             struct instruction_data *data)
5895 {
5896         struct header *h;
5897
5898         CHECK(n_tokens == 3, EINVAL);
5899
5900         strcpy(data->jmp_label, tokens[1]);
5901
5902         h = header_parse(p, tokens[2]);
5903         CHECK(h, EINVAL);
5904
5905         instr->type = INSTR_JMP_INVALID;
5906         instr->jmp.ip = NULL; /* Resolved later. */
5907         instr->jmp.header_id = h->id;
5908         return 0;
5909 }
5910
5911 static int
5912 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5913                         struct action *action,
5914                         char **tokens,
5915                         int n_tokens,
5916                         struct instruction *instr,
5917                         struct instruction_data *data)
5918 {
5919         CHECK(!action, EINVAL);
5920         CHECK(n_tokens == 2, EINVAL);
5921
5922         strcpy(data->jmp_label, tokens[1]);
5923
5924         instr->type = INSTR_JMP_HIT;
5925         instr->jmp.ip = NULL; /* Resolved later. */
5926         return 0;
5927 }
5928
5929 static int
5930 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5931                          struct action *action,
5932                          char **tokens,
5933                          int n_tokens,
5934                          struct instruction *instr,
5935                          struct instruction_data *data)
5936 {
5937         CHECK(!action, EINVAL);
5938         CHECK(n_tokens == 2, EINVAL);
5939
5940         strcpy(data->jmp_label, tokens[1]);
5941
5942         instr->type = INSTR_JMP_MISS;
5943         instr->jmp.ip = NULL; /* Resolved later. */
5944         return 0;
5945 }
5946
5947 static int
5948 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5949                                struct action *action,
5950                                char **tokens,
5951                                int n_tokens,
5952                                struct instruction *instr,
5953                                struct instruction_data *data)
5954 {
5955         struct action *a;
5956
5957         CHECK(!action, EINVAL);
5958         CHECK(n_tokens == 3, EINVAL);
5959
5960         strcpy(data->jmp_label, tokens[1]);
5961
5962         a = action_find(p, tokens[2]);
5963         CHECK(a, EINVAL);
5964
5965         instr->type = INSTR_JMP_ACTION_HIT;
5966         instr->jmp.ip = NULL; /* Resolved later. */
5967         instr->jmp.action_id = a->id;
5968         return 0;
5969 }
5970
5971 static int
5972 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5973                                 struct action *action,
5974                                 char **tokens,
5975                                 int n_tokens,
5976                                 struct instruction *instr,
5977                                 struct instruction_data *data)
5978 {
5979         struct action *a;
5980
5981         CHECK(!action, EINVAL);
5982         CHECK(n_tokens == 3, EINVAL);
5983
5984         strcpy(data->jmp_label, tokens[1]);
5985
5986         a = action_find(p, tokens[2]);
5987         CHECK(a, EINVAL);
5988
5989         instr->type = INSTR_JMP_ACTION_MISS;
5990         instr->jmp.ip = NULL; /* Resolved later. */
5991         instr->jmp.action_id = a->id;
5992         return 0;
5993 }
5994
5995 static int
5996 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5997                        struct action *action,
5998                        char **tokens,
5999                        int n_tokens,
6000                        struct instruction *instr,
6001                        struct instruction_data *data)
6002 {
6003         char *a = tokens[2], *b = tokens[3];
6004         struct field *fa, *fb;
6005         uint64_t b_val;
6006         uint32_t a_struct_id, b_struct_id;
6007
6008         CHECK(n_tokens == 4, EINVAL);
6009
6010         strcpy(data->jmp_label, tokens[1]);
6011
6012         fa = struct_field_parse(p, action, a, &a_struct_id);
6013         CHECK(fa, EINVAL);
6014         CHECK(!fa->var_size, EINVAL);
6015
6016         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
6017         fb = struct_field_parse(p, action, b, &b_struct_id);
6018         if (fb) {
6019                 CHECK(!fb->var_size, EINVAL);
6020
6021                 instr->type = INSTR_JMP_EQ;
6022                 if (a[0] != 'h' && b[0] == 'h')
6023                         instr->type = INSTR_JMP_EQ_MH;
6024                 if (a[0] == 'h' && b[0] != 'h')
6025                         instr->type = INSTR_JMP_EQ_HM;
6026                 if (a[0] == 'h' && b[0] == 'h')
6027                         instr->type = INSTR_JMP_EQ_HH;
6028                 instr->jmp.ip = NULL; /* Resolved later. */
6029
6030                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6031                 instr->jmp.a.n_bits = fa->n_bits;
6032                 instr->jmp.a.offset = fa->offset / 8;
6033                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6034                 instr->jmp.b.n_bits = fb->n_bits;
6035                 instr->jmp.b.offset = fb->offset / 8;
6036                 return 0;
6037         }
6038
6039         /* JMP_EQ_I. */
6040         b_val = strtoull(b, &b, 0);
6041         CHECK(!b[0], EINVAL);
6042
6043         if (a[0] == 'h')
6044                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6045
6046         instr->type = INSTR_JMP_EQ_I;
6047         instr->jmp.ip = NULL; /* Resolved later. */
6048         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6049         instr->jmp.a.n_bits = fa->n_bits;
6050         instr->jmp.a.offset = fa->offset / 8;
6051         instr->jmp.b_val = b_val;
6052         return 0;
6053 }
6054
6055 static int
6056 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6057                         struct action *action,
6058                         char **tokens,
6059                         int n_tokens,
6060                         struct instruction *instr,
6061                         struct instruction_data *data)
6062 {
6063         char *a = tokens[2], *b = tokens[3];
6064         struct field *fa, *fb;
6065         uint64_t b_val;
6066         uint32_t a_struct_id, b_struct_id;
6067
6068         CHECK(n_tokens == 4, EINVAL);
6069
6070         strcpy(data->jmp_label, tokens[1]);
6071
6072         fa = struct_field_parse(p, action, a, &a_struct_id);
6073         CHECK(fa, EINVAL);
6074         CHECK(!fa->var_size, EINVAL);
6075
6076         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
6077         fb = struct_field_parse(p, action, b, &b_struct_id);
6078         if (fb) {
6079                 CHECK(!fb->var_size, EINVAL);
6080
6081                 instr->type = INSTR_JMP_NEQ;
6082                 if (a[0] != 'h' && b[0] == 'h')
6083                         instr->type = INSTR_JMP_NEQ_MH;
6084                 if (a[0] == 'h' && b[0] != 'h')
6085                         instr->type = INSTR_JMP_NEQ_HM;
6086                 if (a[0] == 'h' && b[0] == 'h')
6087                         instr->type = INSTR_JMP_NEQ_HH;
6088                 instr->jmp.ip = NULL; /* Resolved later. */
6089
6090                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6091                 instr->jmp.a.n_bits = fa->n_bits;
6092                 instr->jmp.a.offset = fa->offset / 8;
6093                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6094                 instr->jmp.b.n_bits = fb->n_bits;
6095                 instr->jmp.b.offset = fb->offset / 8;
6096                 return 0;
6097         }
6098
6099         /* JMP_NEQ_I. */
6100         b_val = strtoull(b, &b, 0);
6101         CHECK(!b[0], EINVAL);
6102
6103         if (a[0] == 'h')
6104                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6105
6106         instr->type = INSTR_JMP_NEQ_I;
6107         instr->jmp.ip = NULL; /* Resolved later. */
6108         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6109         instr->jmp.a.n_bits = fa->n_bits;
6110         instr->jmp.a.offset = fa->offset / 8;
6111         instr->jmp.b_val = b_val;
6112         return 0;
6113 }
6114
6115 static int
6116 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6117                        struct action *action,
6118                        char **tokens,
6119                        int n_tokens,
6120                        struct instruction *instr,
6121                        struct instruction_data *data)
6122 {
6123         char *a = tokens[2], *b = tokens[3];
6124         struct field *fa, *fb;
6125         uint64_t b_val;
6126         uint32_t a_struct_id, b_struct_id;
6127
6128         CHECK(n_tokens == 4, EINVAL);
6129
6130         strcpy(data->jmp_label, tokens[1]);
6131
6132         fa = struct_field_parse(p, action, a, &a_struct_id);
6133         CHECK(fa, EINVAL);
6134         CHECK(!fa->var_size, EINVAL);
6135
6136         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6137         fb = struct_field_parse(p, action, b, &b_struct_id);
6138         if (fb) {
6139                 CHECK(!fb->var_size, EINVAL);
6140
6141                 instr->type = INSTR_JMP_LT;
6142                 if (a[0] == 'h' && b[0] != 'h')
6143                         instr->type = INSTR_JMP_LT_HM;
6144                 if (a[0] != 'h' && b[0] == 'h')
6145                         instr->type = INSTR_JMP_LT_MH;
6146                 if (a[0] == 'h' && b[0] == 'h')
6147                         instr->type = INSTR_JMP_LT_HH;
6148                 instr->jmp.ip = NULL; /* Resolved later. */
6149
6150                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6151                 instr->jmp.a.n_bits = fa->n_bits;
6152                 instr->jmp.a.offset = fa->offset / 8;
6153                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6154                 instr->jmp.b.n_bits = fb->n_bits;
6155                 instr->jmp.b.offset = fb->offset / 8;
6156                 return 0;
6157         }
6158
6159         /* JMP_LT_MI, JMP_LT_HI. */
6160         b_val = strtoull(b, &b, 0);
6161         CHECK(!b[0], EINVAL);
6162
6163         instr->type = INSTR_JMP_LT_MI;
6164         if (a[0] == 'h')
6165                 instr->type = INSTR_JMP_LT_HI;
6166         instr->jmp.ip = NULL; /* Resolved later. */
6167
6168         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6169         instr->jmp.a.n_bits = fa->n_bits;
6170         instr->jmp.a.offset = fa->offset / 8;
6171         instr->jmp.b_val = b_val;
6172         return 0;
6173 }
6174
6175 static int
6176 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6177                        struct action *action,
6178                        char **tokens,
6179                        int n_tokens,
6180                        struct instruction *instr,
6181                        struct instruction_data *data)
6182 {
6183         char *a = tokens[2], *b = tokens[3];
6184         struct field *fa, *fb;
6185         uint64_t b_val;
6186         uint32_t a_struct_id, b_struct_id;
6187
6188         CHECK(n_tokens == 4, EINVAL);
6189
6190         strcpy(data->jmp_label, tokens[1]);
6191
6192         fa = struct_field_parse(p, action, a, &a_struct_id);
6193         CHECK(fa, EINVAL);
6194         CHECK(!fa->var_size, EINVAL);
6195
6196         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6197         fb = struct_field_parse(p, action, b, &b_struct_id);
6198         if (fb) {
6199                 CHECK(!fb->var_size, EINVAL);
6200
6201                 instr->type = INSTR_JMP_GT;
6202                 if (a[0] == 'h' && b[0] != 'h')
6203                         instr->type = INSTR_JMP_GT_HM;
6204                 if (a[0] != 'h' && b[0] == 'h')
6205                         instr->type = INSTR_JMP_GT_MH;
6206                 if (a[0] == 'h' && b[0] == 'h')
6207                         instr->type = INSTR_JMP_GT_HH;
6208                 instr->jmp.ip = NULL; /* Resolved later. */
6209
6210                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6211                 instr->jmp.a.n_bits = fa->n_bits;
6212                 instr->jmp.a.offset = fa->offset / 8;
6213                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6214                 instr->jmp.b.n_bits = fb->n_bits;
6215                 instr->jmp.b.offset = fb->offset / 8;
6216                 return 0;
6217         }
6218
6219         /* JMP_GT_MI, JMP_GT_HI. */
6220         b_val = strtoull(b, &b, 0);
6221         CHECK(!b[0], EINVAL);
6222
6223         instr->type = INSTR_JMP_GT_MI;
6224         if (a[0] == 'h')
6225                 instr->type = INSTR_JMP_GT_HI;
6226         instr->jmp.ip = NULL; /* Resolved later. */
6227
6228         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6229         instr->jmp.a.n_bits = fa->n_bits;
6230         instr->jmp.a.offset = fa->offset / 8;
6231         instr->jmp.b_val = b_val;
6232         return 0;
6233 }
6234
6235 static inline void
6236 instr_jmp_exec(struct rte_swx_pipeline *p)
6237 {
6238         struct thread *t = &p->threads[p->thread_id];
6239         struct instruction *ip = t->ip;
6240
6241         TRACE("[Thread %2u] jmp\n", p->thread_id);
6242
6243         thread_ip_set(t, ip->jmp.ip);
6244 }
6245
6246 static inline void
6247 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6248 {
6249         struct thread *t = &p->threads[p->thread_id];
6250         struct instruction *ip = t->ip;
6251         uint32_t header_id = ip->jmp.header_id;
6252
6253         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6254
6255         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6256 }
6257
6258 static inline void
6259 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6260 {
6261         struct thread *t = &p->threads[p->thread_id];
6262         struct instruction *ip = t->ip;
6263         uint32_t header_id = ip->jmp.header_id;
6264
6265         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6266
6267         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6268 }
6269
6270 static inline void
6271 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6272 {
6273         struct thread *t = &p->threads[p->thread_id];
6274         struct instruction *ip = t->ip;
6275         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6276
6277         TRACE("[Thread %2u] jmph\n", p->thread_id);
6278
6279         t->ip = ip_next[t->hit];
6280 }
6281
6282 static inline void
6283 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6284 {
6285         struct thread *t = &p->threads[p->thread_id];
6286         struct instruction *ip = t->ip;
6287         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6288
6289         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6290
6291         t->ip = ip_next[t->hit];
6292 }
6293
6294 static inline void
6295 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6296 {
6297         struct thread *t = &p->threads[p->thread_id];
6298         struct instruction *ip = t->ip;
6299
6300         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6301
6302         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6303 }
6304
6305 static inline void
6306 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6307 {
6308         struct thread *t = &p->threads[p->thread_id];
6309         struct instruction *ip = t->ip;
6310
6311         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6312
6313         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6314 }
6315
6316 static inline void
6317 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6318 {
6319         struct thread *t = &p->threads[p->thread_id];
6320         struct instruction *ip = t->ip;
6321
6322         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6323
6324         JMP_CMP(t, ip, ==);
6325 }
6326
6327 static inline void
6328 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6329 {
6330         struct thread *t = &p->threads[p->thread_id];
6331         struct instruction *ip = t->ip;
6332
6333         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6334
6335         JMP_CMP_MH(t, ip, ==);
6336 }
6337
6338 static inline void
6339 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6340 {
6341         struct thread *t = &p->threads[p->thread_id];
6342         struct instruction *ip = t->ip;
6343
6344         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6345
6346         JMP_CMP_HM(t, ip, ==);
6347 }
6348
6349 static inline void
6350 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6351 {
6352         struct thread *t = &p->threads[p->thread_id];
6353         struct instruction *ip = t->ip;
6354
6355         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6356
6357         JMP_CMP_HH_FAST(t, ip, ==);
6358 }
6359
6360 static inline void
6361 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6362 {
6363         struct thread *t = &p->threads[p->thread_id];
6364         struct instruction *ip = t->ip;
6365
6366         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6367
6368         JMP_CMP_I(t, ip, ==);
6369 }
6370
6371 static inline void
6372 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6373 {
6374         struct thread *t = &p->threads[p->thread_id];
6375         struct instruction *ip = t->ip;
6376
6377         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6378
6379         JMP_CMP(t, ip, !=);
6380 }
6381
6382 static inline void
6383 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6384 {
6385         struct thread *t = &p->threads[p->thread_id];
6386         struct instruction *ip = t->ip;
6387
6388         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6389
6390         JMP_CMP_MH(t, ip, !=);
6391 }
6392
6393 static inline void
6394 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6395 {
6396         struct thread *t = &p->threads[p->thread_id];
6397         struct instruction *ip = t->ip;
6398
6399         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6400
6401         JMP_CMP_HM(t, ip, !=);
6402 }
6403
6404 static inline void
6405 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6406 {
6407         struct thread *t = &p->threads[p->thread_id];
6408         struct instruction *ip = t->ip;
6409
6410         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6411
6412         JMP_CMP_HH_FAST(t, ip, !=);
6413 }
6414
6415 static inline void
6416 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6417 {
6418         struct thread *t = &p->threads[p->thread_id];
6419         struct instruction *ip = t->ip;
6420
6421         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6422
6423         JMP_CMP_I(t, ip, !=);
6424 }
6425
6426 static inline void
6427 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6428 {
6429         struct thread *t = &p->threads[p->thread_id];
6430         struct instruction *ip = t->ip;
6431
6432         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6433
6434         JMP_CMP(t, ip, <);
6435 }
6436
6437 static inline void
6438 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6439 {
6440         struct thread *t = &p->threads[p->thread_id];
6441         struct instruction *ip = t->ip;
6442
6443         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6444
6445         JMP_CMP_MH(t, ip, <);
6446 }
6447
6448 static inline void
6449 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6450 {
6451         struct thread *t = &p->threads[p->thread_id];
6452         struct instruction *ip = t->ip;
6453
6454         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6455
6456         JMP_CMP_HM(t, ip, <);
6457 }
6458
6459 static inline void
6460 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6461 {
6462         struct thread *t = &p->threads[p->thread_id];
6463         struct instruction *ip = t->ip;
6464
6465         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6466
6467         JMP_CMP_HH(t, ip, <);
6468 }
6469
6470 static inline void
6471 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6472 {
6473         struct thread *t = &p->threads[p->thread_id];
6474         struct instruction *ip = t->ip;
6475
6476         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6477
6478         JMP_CMP_MI(t, ip, <);
6479 }
6480
6481 static inline void
6482 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6483 {
6484         struct thread *t = &p->threads[p->thread_id];
6485         struct instruction *ip = t->ip;
6486
6487         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6488
6489         JMP_CMP_HI(t, ip, <);
6490 }
6491
6492 static inline void
6493 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6494 {
6495         struct thread *t = &p->threads[p->thread_id];
6496         struct instruction *ip = t->ip;
6497
6498         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6499
6500         JMP_CMP(t, ip, >);
6501 }
6502
6503 static inline void
6504 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6505 {
6506         struct thread *t = &p->threads[p->thread_id];
6507         struct instruction *ip = t->ip;
6508
6509         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6510
6511         JMP_CMP_MH(t, ip, >);
6512 }
6513
6514 static inline void
6515 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6516 {
6517         struct thread *t = &p->threads[p->thread_id];
6518         struct instruction *ip = t->ip;
6519
6520         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6521
6522         JMP_CMP_HM(t, ip, >);
6523 }
6524
6525 static inline void
6526 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6527 {
6528         struct thread *t = &p->threads[p->thread_id];
6529         struct instruction *ip = t->ip;
6530
6531         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6532
6533         JMP_CMP_HH(t, ip, >);
6534 }
6535
6536 static inline void
6537 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6538 {
6539         struct thread *t = &p->threads[p->thread_id];
6540         struct instruction *ip = t->ip;
6541
6542         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6543
6544         JMP_CMP_MI(t, ip, >);
6545 }
6546
6547 static inline void
6548 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6549 {
6550         struct thread *t = &p->threads[p->thread_id];
6551         struct instruction *ip = t->ip;
6552
6553         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6554
6555         JMP_CMP_HI(t, ip, >);
6556 }
6557
6558 /*
6559  * return.
6560  */
6561 static int
6562 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6563                        struct action *action,
6564                        char **tokens __rte_unused,
6565                        int n_tokens,
6566                        struct instruction *instr,
6567                        struct instruction_data *data __rte_unused)
6568 {
6569         CHECK(action, EINVAL);
6570         CHECK(n_tokens == 1, EINVAL);
6571
6572         instr->type = INSTR_RETURN;
6573         return 0;
6574 }
6575
6576 static inline void
6577 instr_return_exec(struct rte_swx_pipeline *p)
6578 {
6579         struct thread *t = &p->threads[p->thread_id];
6580
6581         TRACE("[Thread %2u] return\n", p->thread_id);
6582
6583         t->ip = t->ret;
6584 }
6585
6586 static int
6587 instr_translate(struct rte_swx_pipeline *p,
6588                 struct action *action,
6589                 char *string,
6590                 struct instruction *instr,
6591                 struct instruction_data *data)
6592 {
6593         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6594         int n_tokens = 0, tpos = 0;
6595
6596         /* Parse the instruction string into tokens. */
6597         for ( ; ; ) {
6598                 char *token;
6599
6600                 token = strtok_r(string, " \t\v", &string);
6601                 if (!token)
6602                         break;
6603
6604                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6605                 CHECK_NAME(token, EINVAL);
6606
6607                 tokens[n_tokens] = token;
6608                 n_tokens++;
6609         }
6610
6611         CHECK(n_tokens, EINVAL);
6612
6613         /* Handle the optional instruction label. */
6614         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6615                 strcpy(data->label, tokens[0]);
6616
6617                 tpos += 2;
6618                 CHECK(n_tokens - tpos, EINVAL);
6619         }
6620
6621         /* Identify the instruction type. */
6622         if (!strcmp(tokens[tpos], "rx"))
6623                 return instr_rx_translate(p,
6624                                           action,
6625                                           &tokens[tpos],
6626                                           n_tokens - tpos,
6627                                           instr,
6628                                           data);
6629
6630         if (!strcmp(tokens[tpos], "tx"))
6631                 return instr_tx_translate(p,
6632                                           action,
6633                                           &tokens[tpos],
6634                                           n_tokens - tpos,
6635                                           instr,
6636                                           data);
6637
6638         if (!strcmp(tokens[tpos], "drop"))
6639                 return instr_drop_translate(p,
6640                                             action,
6641                                             &tokens[tpos],
6642                                             n_tokens - tpos,
6643                                             instr,
6644                                             data);
6645
6646         if (!strcmp(tokens[tpos], "extract"))
6647                 return instr_hdr_extract_translate(p,
6648                                                    action,
6649                                                    &tokens[tpos],
6650                                                    n_tokens - tpos,
6651                                                    instr,
6652                                                    data);
6653
6654         if (!strcmp(tokens[tpos], "lookahead"))
6655                 return instr_hdr_lookahead_translate(p,
6656                                                      action,
6657                                                      &tokens[tpos],
6658                                                      n_tokens - tpos,
6659                                                      instr,
6660                                                      data);
6661
6662         if (!strcmp(tokens[tpos], "emit"))
6663                 return instr_hdr_emit_translate(p,
6664                                                 action,
6665                                                 &tokens[tpos],
6666                                                 n_tokens - tpos,
6667                                                 instr,
6668                                                 data);
6669
6670         if (!strcmp(tokens[tpos], "validate"))
6671                 return instr_hdr_validate_translate(p,
6672                                                     action,
6673                                                     &tokens[tpos],
6674                                                     n_tokens - tpos,
6675                                                     instr,
6676                                                     data);
6677
6678         if (!strcmp(tokens[tpos], "invalidate"))
6679                 return instr_hdr_invalidate_translate(p,
6680                                                       action,
6681                                                       &tokens[tpos],
6682                                                       n_tokens - tpos,
6683                                                       instr,
6684                                                       data);
6685
6686         if (!strcmp(tokens[tpos], "mov"))
6687                 return instr_mov_translate(p,
6688                                            action,
6689                                            &tokens[tpos],
6690                                            n_tokens - tpos,
6691                                            instr,
6692                                            data);
6693
6694         if (!strcmp(tokens[tpos], "add"))
6695                 return instr_alu_add_translate(p,
6696                                                action,
6697                                                &tokens[tpos],
6698                                                n_tokens - tpos,
6699                                                instr,
6700                                                data);
6701
6702         if (!strcmp(tokens[tpos], "sub"))
6703                 return instr_alu_sub_translate(p,
6704                                                action,
6705                                                &tokens[tpos],
6706                                                n_tokens - tpos,
6707                                                instr,
6708                                                data);
6709
6710         if (!strcmp(tokens[tpos], "ckadd"))
6711                 return instr_alu_ckadd_translate(p,
6712                                                  action,
6713                                                  &tokens[tpos],
6714                                                  n_tokens - tpos,
6715                                                  instr,
6716                                                  data);
6717
6718         if (!strcmp(tokens[tpos], "cksub"))
6719                 return instr_alu_cksub_translate(p,
6720                                                  action,
6721                                                  &tokens[tpos],
6722                                                  n_tokens - tpos,
6723                                                  instr,
6724                                                  data);
6725
6726         if (!strcmp(tokens[tpos], "and"))
6727                 return instr_alu_and_translate(p,
6728                                                action,
6729                                                &tokens[tpos],
6730                                                n_tokens - tpos,
6731                                                instr,
6732                                                data);
6733
6734         if (!strcmp(tokens[tpos], "or"))
6735                 return instr_alu_or_translate(p,
6736                                               action,
6737                                               &tokens[tpos],
6738                                               n_tokens - tpos,
6739                                               instr,
6740                                               data);
6741
6742         if (!strcmp(tokens[tpos], "xor"))
6743                 return instr_alu_xor_translate(p,
6744                                                action,
6745                                                &tokens[tpos],
6746                                                n_tokens - tpos,
6747                                                instr,
6748                                                data);
6749
6750         if (!strcmp(tokens[tpos], "shl"))
6751                 return instr_alu_shl_translate(p,
6752                                                action,
6753                                                &tokens[tpos],
6754                                                n_tokens - tpos,
6755                                                instr,
6756                                                data);
6757
6758         if (!strcmp(tokens[tpos], "shr"))
6759                 return instr_alu_shr_translate(p,
6760                                                action,
6761                                                &tokens[tpos],
6762                                                n_tokens - tpos,
6763                                                instr,
6764                                                data);
6765
6766         if (!strcmp(tokens[tpos], "regprefetch"))
6767                 return instr_regprefetch_translate(p,
6768                                                    action,
6769                                                    &tokens[tpos],
6770                                                    n_tokens - tpos,
6771                                                    instr,
6772                                                    data);
6773
6774         if (!strcmp(tokens[tpos], "regrd"))
6775                 return instr_regrd_translate(p,
6776                                              action,
6777                                              &tokens[tpos],
6778                                              n_tokens - tpos,
6779                                              instr,
6780                                              data);
6781
6782         if (!strcmp(tokens[tpos], "regwr"))
6783                 return instr_regwr_translate(p,
6784                                              action,
6785                                              &tokens[tpos],
6786                                              n_tokens - tpos,
6787                                              instr,
6788                                              data);
6789
6790         if (!strcmp(tokens[tpos], "regadd"))
6791                 return instr_regadd_translate(p,
6792                                               action,
6793                                               &tokens[tpos],
6794                                               n_tokens - tpos,
6795                                               instr,
6796                                               data);
6797
6798         if (!strcmp(tokens[tpos], "metprefetch"))
6799                 return instr_metprefetch_translate(p,
6800                                                    action,
6801                                                    &tokens[tpos],
6802                                                    n_tokens - tpos,
6803                                                    instr,
6804                                                    data);
6805
6806         if (!strcmp(tokens[tpos], "meter"))
6807                 return instr_meter_translate(p,
6808                                              action,
6809                                              &tokens[tpos],
6810                                              n_tokens - tpos,
6811                                              instr,
6812                                              data);
6813
6814         if (!strcmp(tokens[tpos], "table"))
6815                 return instr_table_translate(p,
6816                                              action,
6817                                              &tokens[tpos],
6818                                              n_tokens - tpos,
6819                                              instr,
6820                                              data);
6821
6822         if (!strcmp(tokens[tpos], "learn"))
6823                 return instr_learn_translate(p,
6824                                              action,
6825                                              &tokens[tpos],
6826                                              n_tokens - tpos,
6827                                              instr,
6828                                              data);
6829
6830         if (!strcmp(tokens[tpos], "forget"))
6831                 return instr_forget_translate(p,
6832                                               action,
6833                                               &tokens[tpos],
6834                                               n_tokens - tpos,
6835                                               instr,
6836                                               data);
6837
6838         if (!strcmp(tokens[tpos], "extern"))
6839                 return instr_extern_translate(p,
6840                                               action,
6841                                               &tokens[tpos],
6842                                               n_tokens - tpos,
6843                                               instr,
6844                                               data);
6845
6846         if (!strcmp(tokens[tpos], "jmp"))
6847                 return instr_jmp_translate(p,
6848                                            action,
6849                                            &tokens[tpos],
6850                                            n_tokens - tpos,
6851                                            instr,
6852                                            data);
6853
6854         if (!strcmp(tokens[tpos], "jmpv"))
6855                 return instr_jmp_valid_translate(p,
6856                                                  action,
6857                                                  &tokens[tpos],
6858                                                  n_tokens - tpos,
6859                                                  instr,
6860                                                  data);
6861
6862         if (!strcmp(tokens[tpos], "jmpnv"))
6863                 return instr_jmp_invalid_translate(p,
6864                                                    action,
6865                                                    &tokens[tpos],
6866                                                    n_tokens - tpos,
6867                                                    instr,
6868                                                    data);
6869
6870         if (!strcmp(tokens[tpos], "jmph"))
6871                 return instr_jmp_hit_translate(p,
6872                                                action,
6873                                                &tokens[tpos],
6874                                                n_tokens - tpos,
6875                                                instr,
6876                                                data);
6877
6878         if (!strcmp(tokens[tpos], "jmpnh"))
6879                 return instr_jmp_miss_translate(p,
6880                                                 action,
6881                                                 &tokens[tpos],
6882                                                 n_tokens - tpos,
6883                                                 instr,
6884                                                 data);
6885
6886         if (!strcmp(tokens[tpos], "jmpa"))
6887                 return instr_jmp_action_hit_translate(p,
6888                                                       action,
6889                                                       &tokens[tpos],
6890                                                       n_tokens - tpos,
6891                                                       instr,
6892                                                       data);
6893
6894         if (!strcmp(tokens[tpos], "jmpna"))
6895                 return instr_jmp_action_miss_translate(p,
6896                                                        action,
6897                                                        &tokens[tpos],
6898                                                        n_tokens - tpos,
6899                                                        instr,
6900                                                        data);
6901
6902         if (!strcmp(tokens[tpos], "jmpeq"))
6903                 return instr_jmp_eq_translate(p,
6904                                               action,
6905                                               &tokens[tpos],
6906                                               n_tokens - tpos,
6907                                               instr,
6908                                               data);
6909
6910         if (!strcmp(tokens[tpos], "jmpneq"))
6911                 return instr_jmp_neq_translate(p,
6912                                                action,
6913                                                &tokens[tpos],
6914                                                n_tokens - tpos,
6915                                                instr,
6916                                                data);
6917
6918         if (!strcmp(tokens[tpos], "jmplt"))
6919                 return instr_jmp_lt_translate(p,
6920                                               action,
6921                                               &tokens[tpos],
6922                                               n_tokens - tpos,
6923                                               instr,
6924                                               data);
6925
6926         if (!strcmp(tokens[tpos], "jmpgt"))
6927                 return instr_jmp_gt_translate(p,
6928                                               action,
6929                                               &tokens[tpos],
6930                                               n_tokens - tpos,
6931                                               instr,
6932                                               data);
6933
6934         if (!strcmp(tokens[tpos], "return"))
6935                 return instr_return_translate(p,
6936                                               action,
6937                                               &tokens[tpos],
6938                                               n_tokens - tpos,
6939                                               instr,
6940                                               data);
6941
6942         CHECK(0, EINVAL);
6943 }
6944
6945 static struct instruction_data *
6946 label_find(struct instruction_data *data, uint32_t n, const char *label)
6947 {
6948         uint32_t i;
6949
6950         for (i = 0; i < n; i++)
6951                 if (!strcmp(label, data[i].label))
6952                         return &data[i];
6953
6954         return NULL;
6955 }
6956
6957 static uint32_t
6958 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6959 {
6960         uint32_t count = 0, i;
6961
6962         if (!label[0])
6963                 return 0;
6964
6965         for (i = 0; i < n; i++)
6966                 if (!strcmp(label, data[i].jmp_label))
6967                         count++;
6968
6969         return count;
6970 }
6971
6972 static int
6973 instr_label_check(struct instruction_data *instruction_data,
6974                   uint32_t n_instructions)
6975 {
6976         uint32_t i;
6977
6978         /* Check that all instruction labels are unique. */
6979         for (i = 0; i < n_instructions; i++) {
6980                 struct instruction_data *data = &instruction_data[i];
6981                 char *label = data->label;
6982                 uint32_t j;
6983
6984                 if (!label[0])
6985                         continue;
6986
6987                 for (j = i + 1; j < n_instructions; j++)
6988                         CHECK(strcmp(label, data[j].label), EINVAL);
6989         }
6990
6991         /* Get users for each instruction label. */
6992         for (i = 0; i < n_instructions; i++) {
6993                 struct instruction_data *data = &instruction_data[i];
6994                 char *label = data->label;
6995
6996                 data->n_users = label_is_used(instruction_data,
6997                                               n_instructions,
6998                                               label);
6999         }
7000
7001         return 0;
7002 }
7003
7004 static int
7005 instr_jmp_resolve(struct instruction *instructions,
7006                   struct instruction_data *instruction_data,
7007                   uint32_t n_instructions)
7008 {
7009         uint32_t i;
7010
7011         for (i = 0; i < n_instructions; i++) {
7012                 struct instruction *instr = &instructions[i];
7013                 struct instruction_data *data = &instruction_data[i];
7014                 struct instruction_data *found;
7015
7016                 if (!instruction_is_jmp(instr))
7017                         continue;
7018
7019                 found = label_find(instruction_data,
7020                                    n_instructions,
7021                                    data->jmp_label);
7022                 CHECK(found, EINVAL);
7023
7024                 instr->jmp.ip = &instructions[found - instruction_data];
7025         }
7026
7027         return 0;
7028 }
7029
7030 static int
7031 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7032              struct action *a,
7033              struct instruction *instr,
7034              struct instruction_data *data __rte_unused,
7035              uint32_t n_instructions)
7036 {
7037         if (!a) {
7038                 enum instruction_type type;
7039                 uint32_t i;
7040
7041                 /* Check that the first instruction is rx. */
7042                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7043
7044                 /* Check that there is at least one tx instruction. */
7045                 for (i = 0; i < n_instructions; i++) {
7046                         type = instr[i].type;
7047
7048                         if (instruction_is_tx(type))
7049                                 break;
7050                 }
7051                 CHECK(i < n_instructions, EINVAL);
7052
7053                 /* Check that the last instruction is either tx or unconditional
7054                  * jump.
7055                  */
7056                 type = instr[n_instructions - 1].type;
7057                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
7058         }
7059
7060         if (a) {
7061                 enum instruction_type type;
7062                 uint32_t i;
7063
7064                 /* Check that there is at least one return or tx instruction. */
7065                 for (i = 0; i < n_instructions; i++) {
7066                         type = instr[i].type;
7067
7068                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
7069                                 break;
7070                 }
7071                 CHECK(i < n_instructions, EINVAL);
7072         }
7073
7074         return 0;
7075 }
7076
7077 static uint32_t
7078 instr_compact(struct instruction *instructions,
7079               struct instruction_data *instruction_data,
7080               uint32_t n_instructions)
7081 {
7082         uint32_t i, pos = 0;
7083
7084         /* Eliminate the invalid instructions that have been optimized out. */
7085         for (i = 0; i < n_instructions; i++) {
7086                 struct instruction *instr = &instructions[i];
7087                 struct instruction_data *data = &instruction_data[i];
7088
7089                 if (data->invalid)
7090                         continue;
7091
7092                 if (i != pos) {
7093                         memcpy(&instructions[pos], instr, sizeof(*instr));
7094                         memcpy(&instruction_data[pos], data, sizeof(*data));
7095                 }
7096
7097                 pos++;
7098         }
7099
7100         return pos;
7101 }
7102
7103 static int
7104 instr_pattern_extract_many_search(struct instruction *instr,
7105                                   struct instruction_data *data,
7106                                   uint32_t n_instr,
7107                                   uint32_t *n_pattern_instr)
7108 {
7109         uint32_t i;
7110
7111         for (i = 0; i < n_instr; i++) {
7112                 if (data[i].invalid)
7113                         break;
7114
7115                 if (instr[i].type != INSTR_HDR_EXTRACT)
7116                         break;
7117
7118                 if (i == RTE_DIM(instr->io.hdr.header_id))
7119                         break;
7120
7121                 if (i && data[i].n_users)
7122                         break;
7123         }
7124
7125         if (i < 2)
7126                 return 0;
7127
7128         *n_pattern_instr = i;
7129         return 1;
7130 }
7131
7132 static void
7133 instr_pattern_extract_many_replace(struct instruction *instr,
7134                                    struct instruction_data *data,
7135                                    uint32_t n_instr)
7136 {
7137         uint32_t i;
7138
7139         for (i = 1; i < n_instr; i++) {
7140                 instr[0].type++;
7141                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7142                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7143                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7144
7145                 data[i].invalid = 1;
7146         }
7147 }
7148
7149 static uint32_t
7150 instr_pattern_extract_many_optimize(struct instruction *instructions,
7151                                     struct instruction_data *instruction_data,
7152                                     uint32_t n_instructions)
7153 {
7154         uint32_t i;
7155
7156         for (i = 0; i < n_instructions; ) {
7157                 struct instruction *instr = &instructions[i];
7158                 struct instruction_data *data = &instruction_data[i];
7159                 uint32_t n_instr = 0;
7160                 int detected;
7161
7162                 /* Extract many. */
7163                 detected = instr_pattern_extract_many_search(instr,
7164                                                              data,
7165                                                              n_instructions - i,
7166                                                              &n_instr);
7167                 if (detected) {
7168                         instr_pattern_extract_many_replace(instr,
7169                                                            data,
7170                                                            n_instr);
7171                         i += n_instr;
7172                         continue;
7173                 }
7174
7175                 /* No pattern starting at the current instruction. */
7176                 i++;
7177         }
7178
7179         /* Eliminate the invalid instructions that have been optimized out. */
7180         n_instructions = instr_compact(instructions,
7181                                        instruction_data,
7182                                        n_instructions);
7183
7184         return n_instructions;
7185 }
7186
7187 static int
7188 instr_pattern_emit_many_tx_search(struct instruction *instr,
7189                                   struct instruction_data *data,
7190                                   uint32_t n_instr,
7191                                   uint32_t *n_pattern_instr)
7192 {
7193         uint32_t i;
7194
7195         for (i = 0; i < n_instr; i++) {
7196                 if (data[i].invalid)
7197                         break;
7198
7199                 if (instr[i].type != INSTR_HDR_EMIT)
7200                         break;
7201
7202                 if (i == RTE_DIM(instr->io.hdr.header_id))
7203                         break;
7204
7205                 if (i && data[i].n_users)
7206                         break;
7207         }
7208
7209         if (!i)
7210                 return 0;
7211
7212         if (!instruction_is_tx(instr[i].type))
7213                 return 0;
7214
7215         if (data[i].n_users)
7216                 return 0;
7217
7218         i++;
7219
7220         *n_pattern_instr = i;
7221         return 1;
7222 }
7223
7224 static void
7225 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7226                                    struct instruction_data *data,
7227                                    uint32_t n_instr)
7228 {
7229         uint32_t i;
7230
7231         /* Any emit instruction in addition to the first one. */
7232         for (i = 1; i < n_instr - 1; i++) {
7233                 instr[0].type++;
7234                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7235                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7236                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7237
7238                 data[i].invalid = 1;
7239         }
7240
7241         /* The TX instruction is the last one in the pattern. */
7242         instr[0].type++;
7243         instr[0].io.io.offset = instr[i].io.io.offset;
7244         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7245         data[i].invalid = 1;
7246 }
7247
7248 static uint32_t
7249 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7250                                     struct instruction_data *instruction_data,
7251                                     uint32_t n_instructions)
7252 {
7253         uint32_t i;
7254
7255         for (i = 0; i < n_instructions; ) {
7256                 struct instruction *instr = &instructions[i];
7257                 struct instruction_data *data = &instruction_data[i];
7258                 uint32_t n_instr = 0;
7259                 int detected;
7260
7261                 /* Emit many + TX. */
7262                 detected = instr_pattern_emit_many_tx_search(instr,
7263                                                              data,
7264                                                              n_instructions - i,
7265                                                              &n_instr);
7266                 if (detected) {
7267                         instr_pattern_emit_many_tx_replace(instr,
7268                                                            data,
7269                                                            n_instr);
7270                         i += n_instr;
7271                         continue;
7272                 }
7273
7274                 /* No pattern starting at the current instruction. */
7275                 i++;
7276         }
7277
7278         /* Eliminate the invalid instructions that have been optimized out. */
7279         n_instructions = instr_compact(instructions,
7280                                        instruction_data,
7281                                        n_instructions);
7282
7283         return n_instructions;
7284 }
7285
7286 static uint32_t
7287 action_arg_src_mov_count(struct action *a,
7288                          uint32_t arg_id,
7289                          struct instruction *instructions,
7290                          struct instruction_data *instruction_data,
7291                          uint32_t n_instructions);
7292
7293 static int
7294 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7295                                       struct action *a,
7296                                       struct instruction *instr,
7297                                       struct instruction_data *data,
7298                                       uint32_t n_instr,
7299                                       struct instruction *instructions,
7300                                       struct instruction_data *instruction_data,
7301                                       uint32_t n_instructions,
7302                                       uint32_t *n_pattern_instr)
7303 {
7304         struct header *h;
7305         uint32_t src_field_id, i, j;
7306
7307         /* Prerequisites. */
7308         if (!a || !a->st)
7309                 return 0;
7310
7311         /* First instruction: MOV_HM. */
7312         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7313                 return 0;
7314
7315         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7316         if (!h || h->st->var_size)
7317                 return 0;
7318
7319         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7320                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7321                         break;
7322
7323         if (src_field_id == a->st->n_fields)
7324                 return 0;
7325
7326         if (instr[0].mov.dst.offset ||
7327             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7328             instr[0].mov.src.struct_id ||
7329             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7330             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7331                 return 0;
7332
7333         if ((n_instr < h->st->n_fields + 1) ||
7334              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7335                 return 0;
7336
7337         /* Subsequent instructions: MOV_HM. */
7338         for (i = 1; i < h->st->n_fields; i++)
7339                 if (data[i].invalid ||
7340                     data[i].n_users ||
7341                     (instr[i].type != INSTR_MOV_HM) ||
7342                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7343                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7344                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7345                     instr[i].mov.src.struct_id ||
7346                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7347                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7348                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7349                         return 0;
7350
7351         /* Last instruction: HDR_VALIDATE. */
7352         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7353             (instr[i].valid.header_id != h->id))
7354                 return 0;
7355
7356         /* Check that none of the action args that are used as source for this
7357          * DMA transfer are not used as source in any other mov instruction.
7358          */
7359         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7360                 uint32_t n_users;
7361
7362                 n_users = action_arg_src_mov_count(a,
7363                                                    j,
7364                                                    instructions,
7365                                                    instruction_data,
7366                                                    n_instructions);
7367                 if (n_users > 1)
7368                         return 0;
7369         }
7370
7371         *n_pattern_instr = 1 + i;
7372         return 1;
7373 }
7374
7375 static void
7376 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7377                                        struct action *a,
7378                                        struct instruction *instr,
7379                                        struct instruction_data *data,
7380                                        uint32_t n_instr)
7381 {
7382         struct header *h;
7383         uint32_t src_field_id, src_offset, i;
7384
7385         /* Read from the instructions before they are modified. */
7386         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7387         if (!h)
7388                 return;
7389
7390         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7391                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7392                         break;
7393
7394         if (src_field_id == a->st->n_fields)
7395                 return;
7396
7397         src_offset = instr[0].mov.src.offset;
7398
7399         /* Modify the instructions. */
7400         instr[0].type = INSTR_DMA_HT;
7401         instr[0].dma.dst.header_id[0] = h->id;
7402         instr[0].dma.dst.struct_id[0] = h->struct_id;
7403         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7404         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7405
7406         for (i = 1; i < n_instr; i++)
7407                 data[i].invalid = 1;
7408
7409         /* Update the endianness of the action arguments to header endianness. */
7410         for (i = 0; i < h->st->n_fields; i++)
7411                 a->args_endianness[src_field_id + i] = 1;
7412 }
7413
7414 static uint32_t
7415 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7416                                         struct action *a,
7417                                         struct instruction *instructions,
7418                                         struct instruction_data *instruction_data,
7419                                         uint32_t n_instructions)
7420 {
7421         uint32_t i;
7422
7423         if (!a || !a->st)
7424                 return n_instructions;
7425
7426         for (i = 0; i < n_instructions; ) {
7427                 struct instruction *instr = &instructions[i];
7428                 struct instruction_data *data = &instruction_data[i];
7429                 uint32_t n_instr = 0;
7430                 int detected;
7431
7432                 /* Mov all + validate. */
7433                 detected = instr_pattern_mov_all_validate_search(p,
7434                                                                  a,
7435                                                                  instr,
7436                                                                  data,
7437                                                                  n_instructions - i,
7438                                                                  instructions,
7439                                                                  instruction_data,
7440                                                                  n_instructions,
7441                                                                  &n_instr);
7442                 if (detected) {
7443                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7444                         i += n_instr;
7445                         continue;
7446                 }
7447
7448                 /* No pattern starting at the current instruction. */
7449                 i++;
7450         }
7451
7452         /* Eliminate the invalid instructions that have been optimized out. */
7453         n_instructions = instr_compact(instructions,
7454                                        instruction_data,
7455                                        n_instructions);
7456
7457         return n_instructions;
7458 }
7459
7460 static int
7461 instr_pattern_dma_many_search(struct instruction *instr,
7462                               struct instruction_data *data,
7463                               uint32_t n_instr,
7464                               uint32_t *n_pattern_instr)
7465 {
7466         uint32_t i;
7467
7468         for (i = 0; i < n_instr; i++) {
7469                 if (data[i].invalid)
7470                         break;
7471
7472                 if (instr[i].type != INSTR_DMA_HT)
7473                         break;
7474
7475                 if (i == RTE_DIM(instr->dma.dst.header_id))
7476                         break;
7477
7478                 if (i && data[i].n_users)
7479                         break;
7480         }
7481
7482         if (i < 2)
7483                 return 0;
7484
7485         *n_pattern_instr = i;
7486         return 1;
7487 }
7488
7489 static void
7490 instr_pattern_dma_many_replace(struct instruction *instr,
7491                                struct instruction_data *data,
7492                                uint32_t n_instr)
7493 {
7494         uint32_t i;
7495
7496         for (i = 1; i < n_instr; i++) {
7497                 instr[0].type++;
7498                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7499                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7500                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7501                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7502
7503                 data[i].invalid = 1;
7504         }
7505 }
7506
7507 static uint32_t
7508 instr_pattern_dma_many_optimize(struct instruction *instructions,
7509                struct instruction_data *instruction_data,
7510                uint32_t n_instructions)
7511 {
7512         uint32_t i;
7513
7514         for (i = 0; i < n_instructions; ) {
7515                 struct instruction *instr = &instructions[i];
7516                 struct instruction_data *data = &instruction_data[i];
7517                 uint32_t n_instr = 0;
7518                 int detected;
7519
7520                 /* DMA many. */
7521                 detected = instr_pattern_dma_many_search(instr,
7522                                                          data,
7523                                                          n_instructions - i,
7524                                                          &n_instr);
7525                 if (detected) {
7526                         instr_pattern_dma_many_replace(instr, data, n_instr);
7527                         i += n_instr;
7528                         continue;
7529                 }
7530
7531                 /* No pattern starting at the current instruction. */
7532                 i++;
7533         }
7534
7535         /* Eliminate the invalid instructions that have been optimized out. */
7536         n_instructions = instr_compact(instructions,
7537                                        instruction_data,
7538                                        n_instructions);
7539
7540         return n_instructions;
7541 }
7542
7543 static uint32_t
7544 instr_optimize(struct rte_swx_pipeline *p,
7545                struct action *a,
7546                struct instruction *instructions,
7547                struct instruction_data *instruction_data,
7548                uint32_t n_instructions)
7549 {
7550         /* Extract many. */
7551         n_instructions = instr_pattern_extract_many_optimize(instructions,
7552                                                              instruction_data,
7553                                                              n_instructions);
7554
7555         /* Emit many + TX. */
7556         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7557                                                              instruction_data,
7558                                                              n_instructions);
7559
7560         /* Mov all + validate. */
7561         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7562                                                                  a,
7563                                                                  instructions,
7564                                                                  instruction_data,
7565                                                                  n_instructions);
7566
7567         /* DMA many. */
7568         n_instructions = instr_pattern_dma_many_optimize(instructions,
7569                                                          instruction_data,
7570                                                          n_instructions);
7571
7572         return n_instructions;
7573 }
7574
7575 static int
7576 instruction_config(struct rte_swx_pipeline *p,
7577                    struct action *a,
7578                    const char **instructions,
7579                    uint32_t n_instructions)
7580 {
7581         struct instruction *instr = NULL;
7582         struct instruction_data *data = NULL;
7583         int err = 0;
7584         uint32_t i;
7585
7586         CHECK(n_instructions, EINVAL);
7587         CHECK(instructions, EINVAL);
7588         for (i = 0; i < n_instructions; i++)
7589                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7590
7591         /* Memory allocation. */
7592         instr = calloc(n_instructions, sizeof(struct instruction));
7593         if (!instr) {
7594                 err = -ENOMEM;
7595                 goto error;
7596         }
7597
7598         data = calloc(n_instructions, sizeof(struct instruction_data));
7599         if (!data) {
7600                 err = -ENOMEM;
7601                 goto error;
7602         }
7603
7604         for (i = 0; i < n_instructions; i++) {
7605                 char *string = strdup(instructions[i]);
7606                 if (!string) {
7607                         err = -ENOMEM;
7608                         goto error;
7609                 }
7610
7611                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7612                 if (err) {
7613                         free(string);
7614                         goto error;
7615                 }
7616
7617                 free(string);
7618         }
7619
7620         err = instr_label_check(data, n_instructions);
7621         if (err)
7622                 goto error;
7623
7624         err = instr_verify(p, a, instr, data, n_instructions);
7625         if (err)
7626                 goto error;
7627
7628         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7629
7630         err = instr_jmp_resolve(instr, data, n_instructions);
7631         if (err)
7632                 goto error;
7633
7634         if (a) {
7635                 a->instructions = instr;
7636                 a->n_instructions = n_instructions;
7637         } else {
7638                 p->instructions = instr;
7639                 p->n_instructions = n_instructions;
7640         }
7641
7642         free(data);
7643         return 0;
7644
7645 error:
7646         free(data);
7647         free(instr);
7648         return err;
7649 }
7650
7651 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7652
7653 static instr_exec_t instruction_table[] = {
7654         [INSTR_RX] = instr_rx_exec,
7655         [INSTR_TX] = instr_tx_exec,
7656         [INSTR_TX_I] = instr_tx_i_exec,
7657
7658         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7659         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7660         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7661         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7662         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7663         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7664         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7665         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7666         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7667         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7668
7669         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7670         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7671         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7672         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7673         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7674         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7675         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7676         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7677         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7678
7679         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7680         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7681
7682         [INSTR_MOV] = instr_mov_exec,
7683         [INSTR_MOV_MH] = instr_mov_mh_exec,
7684         [INSTR_MOV_HM] = instr_mov_hm_exec,
7685         [INSTR_MOV_HH] = instr_mov_hh_exec,
7686         [INSTR_MOV_I] = instr_mov_i_exec,
7687
7688         [INSTR_DMA_HT] = instr_dma_ht_exec,
7689         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7690         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7691         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7692         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7693         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7694         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7695         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7696
7697         [INSTR_ALU_ADD] = instr_alu_add_exec,
7698         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7699         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7700         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7701         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7702         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7703
7704         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7705         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7706         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7707         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7708         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7709         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7710
7711         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7712         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7713         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7714         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7715
7716         [INSTR_ALU_AND] = instr_alu_and_exec,
7717         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7718         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7719         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7720         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7721
7722         [INSTR_ALU_OR] = instr_alu_or_exec,
7723         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7724         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7725         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7726         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7727
7728         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7729         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7730         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7731         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7732         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7733
7734         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7735         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7736         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7737         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7738         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7739         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7740
7741         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7742         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7743         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7744         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7745         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7746         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7747
7748         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7749         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7750         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7751
7752         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7753         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7754         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7755         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7756         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7757         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7758
7759         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7760         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7761         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7762         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7763         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7764         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7765         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7766         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7767         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7768
7769         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7770         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7771         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7772         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7773         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7774         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7775         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7776         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7777         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7778
7779         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7780         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7781         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7782
7783         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7784         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7785         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7786         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7787         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7788         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7789         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7790         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7791         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7792         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7793         [INSTR_METER_IMM] = instr_meter_imm_exec,
7794         [INSTR_METER_IMI] = instr_meter_imi_exec,
7795
7796         [INSTR_TABLE] = instr_table_exec,
7797         [INSTR_SELECTOR] = instr_selector_exec,
7798         [INSTR_LEARNER] = instr_learner_exec,
7799         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7800         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7801         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7802         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7803
7804         [INSTR_JMP] = instr_jmp_exec,
7805         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7806         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7807         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7808         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7809         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7810         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7811
7812         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7813         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7814         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7815         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7816         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7817
7818         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7819         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7820         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7821         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7822         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7823
7824         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7825         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7826         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7827         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7828         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7829         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7830
7831         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7832         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7833         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7834         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7835         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7836         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7837
7838         [INSTR_RETURN] = instr_return_exec,
7839 };
7840
7841 static inline void
7842 instr_exec(struct rte_swx_pipeline *p)
7843 {
7844         struct thread *t = &p->threads[p->thread_id];
7845         struct instruction *ip = t->ip;
7846         instr_exec_t instr = instruction_table[ip->type];
7847
7848         instr(p);
7849 }
7850
7851 /*
7852  * Action.
7853  */
7854 static struct action *
7855 action_find(struct rte_swx_pipeline *p, const char *name)
7856 {
7857         struct action *elem;
7858
7859         if (!name)
7860                 return NULL;
7861
7862         TAILQ_FOREACH(elem, &p->actions, node)
7863                 if (strcmp(elem->name, name) == 0)
7864                         return elem;
7865
7866         return NULL;
7867 }
7868
7869 static struct action *
7870 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7871 {
7872         struct action *action = NULL;
7873
7874         TAILQ_FOREACH(action, &p->actions, node)
7875                 if (action->id == id)
7876                         return action;
7877
7878         return NULL;
7879 }
7880
7881 static struct field *
7882 action_field_find(struct action *a, const char *name)
7883 {
7884         return a->st ? struct_type_field_find(a->st, name) : NULL;
7885 }
7886
7887 static struct field *
7888 action_field_parse(struct action *action, const char *name)
7889 {
7890         if (name[0] != 't' || name[1] != '.')
7891                 return NULL;
7892
7893         return action_field_find(action, &name[2]);
7894 }
7895
7896 static int
7897 action_has_nbo_args(struct action *a)
7898 {
7899         uint32_t i;
7900
7901         /* Return if the action does not have any args. */
7902         if (!a->st)
7903                 return 0; /* FALSE */
7904
7905         for (i = 0; i < a->st->n_fields; i++)
7906                 if (a->args_endianness[i])
7907                         return 1; /* TRUE */
7908
7909         return 0; /* FALSE */
7910 }
7911
7912 static int
7913 action_does_learning(struct action *a)
7914 {
7915         uint32_t i;
7916
7917         for (i = 0; i < a->n_instructions; i++)
7918                 switch (a->instructions[i].type) {
7919                 case INSTR_LEARNER_LEARN:
7920                         return 1; /* TRUE */
7921
7922                 case INSTR_LEARNER_FORGET:
7923                         return 1; /* TRUE */
7924
7925                 default:
7926                         continue;
7927                 }
7928
7929         return 0; /* FALSE */
7930 }
7931
7932 int
7933 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7934                                const char *name,
7935                                const char *args_struct_type_name,
7936                                const char **instructions,
7937                                uint32_t n_instructions)
7938 {
7939         struct struct_type *args_struct_type = NULL;
7940         struct action *a;
7941         int err;
7942
7943         CHECK(p, EINVAL);
7944
7945         CHECK_NAME(name, EINVAL);
7946         CHECK(!action_find(p, name), EEXIST);
7947
7948         if (args_struct_type_name) {
7949                 CHECK_NAME(args_struct_type_name, EINVAL);
7950                 args_struct_type = struct_type_find(p, args_struct_type_name);
7951                 CHECK(args_struct_type, EINVAL);
7952                 CHECK(!args_struct_type->var_size, EINVAL);
7953         }
7954
7955         /* Node allocation. */
7956         a = calloc(1, sizeof(struct action));
7957         CHECK(a, ENOMEM);
7958         if (args_struct_type) {
7959                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7960                 if (!a->args_endianness) {
7961                         free(a);
7962                         CHECK(0, ENOMEM);
7963                 }
7964         }
7965
7966         /* Node initialization. */
7967         strcpy(a->name, name);
7968         a->st = args_struct_type;
7969         a->id = p->n_actions;
7970
7971         /* Instruction translation. */
7972         err = instruction_config(p, a, instructions, n_instructions);
7973         if (err) {
7974                 free(a->args_endianness);
7975                 free(a);
7976                 return err;
7977         }
7978
7979         /* Node add to tailq. */
7980         TAILQ_INSERT_TAIL(&p->actions, a, node);
7981         p->n_actions++;
7982
7983         return 0;
7984 }
7985
7986 static int
7987 action_build(struct rte_swx_pipeline *p)
7988 {
7989         struct action *action;
7990
7991         p->action_instructions = calloc(p->n_actions,
7992                                         sizeof(struct instruction *));
7993         CHECK(p->action_instructions, ENOMEM);
7994
7995         TAILQ_FOREACH(action, &p->actions, node)
7996                 p->action_instructions[action->id] = action->instructions;
7997
7998         return 0;
7999 }
8000
8001 static void
8002 action_build_free(struct rte_swx_pipeline *p)
8003 {
8004         free(p->action_instructions);
8005         p->action_instructions = NULL;
8006 }
8007
8008 static void
8009 action_free(struct rte_swx_pipeline *p)
8010 {
8011         action_build_free(p);
8012
8013         for ( ; ; ) {
8014                 struct action *action;
8015
8016                 action = TAILQ_FIRST(&p->actions);
8017                 if (!action)
8018                         break;
8019
8020                 TAILQ_REMOVE(&p->actions, action, node);
8021                 free(action->instructions);
8022                 free(action);
8023         }
8024 }
8025
8026 static uint32_t
8027 action_arg_src_mov_count(struct action *a,
8028                          uint32_t arg_id,
8029                          struct instruction *instructions,
8030                          struct instruction_data *instruction_data,
8031                          uint32_t n_instructions)
8032 {
8033         uint32_t offset, n_users = 0, i;
8034
8035         if (!a->st ||
8036             (arg_id >= a->st->n_fields) ||
8037             !instructions ||
8038             !instruction_data ||
8039             !n_instructions)
8040                 return 0;
8041
8042         offset = a->st->fields[arg_id].offset / 8;
8043
8044         for (i = 0; i < n_instructions; i++) {
8045                 struct instruction *instr = &instructions[i];
8046                 struct instruction_data *data = &instruction_data[i];
8047
8048                 if (data->invalid ||
8049                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
8050                     instr->mov.src.struct_id ||
8051                     (instr->mov.src.offset != offset))
8052                         continue;
8053
8054                 n_users++;
8055         }
8056
8057         return n_users;
8058 }
8059
8060 /*
8061  * Table.
8062  */
8063 static struct table_type *
8064 table_type_find(struct rte_swx_pipeline *p, const char *name)
8065 {
8066         struct table_type *elem;
8067
8068         TAILQ_FOREACH(elem, &p->table_types, node)
8069                 if (strcmp(elem->name, name) == 0)
8070                         return elem;
8071
8072         return NULL;
8073 }
8074
8075 static struct table_type *
8076 table_type_resolve(struct rte_swx_pipeline *p,
8077                    const char *recommended_type_name,
8078                    enum rte_swx_table_match_type match_type)
8079 {
8080         struct table_type *elem;
8081
8082         /* Only consider the recommended type if the match type is correct. */
8083         if (recommended_type_name)
8084                 TAILQ_FOREACH(elem, &p->table_types, node)
8085                         if (!strcmp(elem->name, recommended_type_name) &&
8086                             (elem->match_type == match_type))
8087                                 return elem;
8088
8089         /* Ignore the recommended type and get the first element with this match
8090          * type.
8091          */
8092         TAILQ_FOREACH(elem, &p->table_types, node)
8093                 if (elem->match_type == match_type)
8094                         return elem;
8095
8096         return NULL;
8097 }
8098
8099 static struct table *
8100 table_find(struct rte_swx_pipeline *p, const char *name)
8101 {
8102         struct table *elem;
8103
8104         TAILQ_FOREACH(elem, &p->tables, node)
8105                 if (strcmp(elem->name, name) == 0)
8106                         return elem;
8107
8108         return NULL;
8109 }
8110
8111 static struct table *
8112 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8113 {
8114         struct table *table = NULL;
8115
8116         TAILQ_FOREACH(table, &p->tables, node)
8117                 if (table->id == id)
8118                         return table;
8119
8120         return NULL;
8121 }
8122
8123 int
8124 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8125                                      const char *name,
8126                                      enum rte_swx_table_match_type match_type,
8127                                      struct rte_swx_table_ops *ops)
8128 {
8129         struct table_type *elem;
8130
8131         CHECK(p, EINVAL);
8132
8133         CHECK_NAME(name, EINVAL);
8134         CHECK(!table_type_find(p, name), EEXIST);
8135
8136         CHECK(ops, EINVAL);
8137         CHECK(ops->create, EINVAL);
8138         CHECK(ops->lkp, EINVAL);
8139         CHECK(ops->free, EINVAL);
8140
8141         /* Node allocation. */
8142         elem = calloc(1, sizeof(struct table_type));
8143         CHECK(elem, ENOMEM);
8144
8145         /* Node initialization. */
8146         strcpy(elem->name, name);
8147         elem->match_type = match_type;
8148         memcpy(&elem->ops, ops, sizeof(*ops));
8149
8150         /* Node add to tailq. */
8151         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8152
8153         return 0;
8154 }
8155
8156 static int
8157 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8158                          uint32_t n_fields,
8159                          enum rte_swx_table_match_type *match_type)
8160 {
8161         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
8162
8163         for (i = 0; i < n_fields; i++) {
8164                 struct rte_swx_match_field_params  *f = &fields[i];
8165
8166                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
8167                         n_fields_em++;
8168
8169                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
8170                         n_fields_lpm++;
8171         }
8172
8173         if ((n_fields_lpm > 1) ||
8174             (n_fields_lpm && (n_fields_em != n_fields - 1)))
8175                 return -EINVAL;
8176
8177         *match_type = (n_fields_em == n_fields) ?
8178                        RTE_SWX_TABLE_MATCH_EXACT :
8179                        RTE_SWX_TABLE_MATCH_WILDCARD;
8180
8181         return 0;
8182 }
8183
8184 static int
8185 table_match_fields_check(struct rte_swx_pipeline *p,
8186                          struct rte_swx_pipeline_table_params *params,
8187                          struct header **header)
8188 {
8189         struct header *h0 = NULL;
8190         struct field *hf, *mf;
8191         uint32_t *offset = NULL, i;
8192         int status = 0;
8193
8194         /* Return if no match fields. */
8195         if (!params->n_fields) {
8196                 if (params->fields) {
8197                         status = -EINVAL;
8198                         goto end;
8199                 }
8200
8201                 if (header)
8202                         *header = NULL;
8203
8204                 return 0;
8205         }
8206
8207         /* Memory allocation. */
8208         offset = calloc(params->n_fields, sizeof(uint32_t));
8209         if (!offset) {
8210                 status = -ENOMEM;
8211                 goto end;
8212         }
8213
8214         /* Check that all the match fields belong to either the same header or
8215          * to the meta-data.
8216          */
8217         hf = header_field_parse(p, params->fields[0].name, &h0);
8218         mf = metadata_field_parse(p, params->fields[0].name);
8219         if ((!hf && !mf) || (hf && hf->var_size)) {
8220                 status = -EINVAL;
8221                 goto end;
8222         }
8223
8224         offset[0] = h0 ? hf->offset : mf->offset;
8225
8226         for (i = 1; i < params->n_fields; i++)
8227                 if (h0) {
8228                         struct header *h;
8229
8230                         hf = header_field_parse(p, params->fields[i].name, &h);
8231                         if (!hf || (h->id != h0->id) || hf->var_size) {
8232                                 status = -EINVAL;
8233                                 goto end;
8234                         }
8235
8236                         offset[i] = hf->offset;
8237                 } else {
8238                         mf = metadata_field_parse(p, params->fields[i].name);
8239                         if (!mf) {
8240                                 status = -EINVAL;
8241                                 goto end;
8242                         }
8243
8244                         offset[i] = mf->offset;
8245                 }
8246
8247         /* Check that there are no duplicated match fields. */
8248         for (i = 0; i < params->n_fields; i++) {
8249                 uint32_t j;
8250
8251                 for (j = 0; j < i; j++)
8252                         if (offset[j] == offset[i]) {
8253                                 status = -EINVAL;
8254                                 goto end;
8255                         }
8256         }
8257
8258         /* Return. */
8259         if (header)
8260                 *header = h0;
8261
8262 end:
8263         free(offset);
8264         return status;
8265 }
8266
8267 int
8268 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8269                               const char *name,
8270                               struct rte_swx_pipeline_table_params *params,
8271                               const char *recommended_table_type_name,
8272                               const char *args,
8273                               uint32_t size)
8274 {
8275         struct table_type *type;
8276         struct table *t;
8277         struct action *default_action;
8278         struct header *header = NULL;
8279         uint32_t action_data_size_max = 0, i;
8280         int status = 0;
8281
8282         CHECK(p, EINVAL);
8283
8284         CHECK_NAME(name, EINVAL);
8285         CHECK(!table_find(p, name), EEXIST);
8286         CHECK(!selector_find(p, name), EEXIST);
8287         CHECK(!learner_find(p, name), EEXIST);
8288
8289         CHECK(params, EINVAL);
8290
8291         /* Match checks. */
8292         status = table_match_fields_check(p, params, &header);
8293         if (status)
8294                 return status;
8295
8296         /* Action checks. */
8297         CHECK(params->n_actions, EINVAL);
8298         CHECK(params->action_names, EINVAL);
8299         for (i = 0; i < params->n_actions; i++) {
8300                 const char *action_name = params->action_names[i];
8301                 struct action *a;
8302                 uint32_t action_data_size;
8303
8304                 CHECK_NAME(action_name, EINVAL);
8305
8306                 a = action_find(p, action_name);
8307                 CHECK(a, EINVAL);
8308                 CHECK(!action_does_learning(a), EINVAL);
8309
8310                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8311                 if (action_data_size > action_data_size_max)
8312                         action_data_size_max = action_data_size;
8313         }
8314
8315         CHECK_NAME(params->default_action_name, EINVAL);
8316         for (i = 0; i < p->n_actions; i++)
8317                 if (!strcmp(params->action_names[i],
8318                             params->default_action_name))
8319                         break;
8320         CHECK(i < params->n_actions, EINVAL);
8321         default_action = action_find(p, params->default_action_name);
8322         CHECK((default_action->st && params->default_action_data) ||
8323               !params->default_action_data, EINVAL);
8324
8325         /* Table type checks. */
8326         if (recommended_table_type_name)
8327                 CHECK_NAME(recommended_table_type_name, EINVAL);
8328
8329         if (params->n_fields) {
8330                 enum rte_swx_table_match_type match_type;
8331
8332                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8333                 if (status)
8334                         return status;
8335
8336                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8337                 CHECK(type, EINVAL);
8338         } else {
8339                 type = NULL;
8340         }
8341
8342         /* Memory allocation. */
8343         t = calloc(1, sizeof(struct table));
8344         CHECK(t, ENOMEM);
8345
8346         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8347         if (!t->fields) {
8348                 free(t);
8349                 CHECK(0, ENOMEM);
8350         }
8351
8352         t->actions = calloc(params->n_actions, sizeof(struct action *));
8353         if (!t->actions) {
8354                 free(t->fields);
8355                 free(t);
8356                 CHECK(0, ENOMEM);
8357         }
8358
8359         if (action_data_size_max) {
8360                 t->default_action_data = calloc(1, action_data_size_max);
8361                 if (!t->default_action_data) {
8362                         free(t->actions);
8363                         free(t->fields);
8364                         free(t);
8365                         CHECK(0, ENOMEM);
8366                 }
8367         }
8368
8369         /* Node initialization. */
8370         strcpy(t->name, name);
8371         if (args && args[0])
8372                 strcpy(t->args, args);
8373         t->type = type;
8374
8375         for (i = 0; i < params->n_fields; i++) {
8376                 struct rte_swx_match_field_params *field = &params->fields[i];
8377                 struct match_field *f = &t->fields[i];
8378
8379                 f->match_type = field->match_type;
8380                 f->field = header ?
8381                         header_field_parse(p, field->name, NULL) :
8382                         metadata_field_parse(p, field->name);
8383         }
8384         t->n_fields = params->n_fields;
8385         t->header = header;
8386
8387         for (i = 0; i < params->n_actions; i++)
8388                 t->actions[i] = action_find(p, params->action_names[i]);
8389         t->default_action = default_action;
8390         if (default_action->st)
8391                 memcpy(t->default_action_data,
8392                        params->default_action_data,
8393                        default_action->st->n_bits / 8);
8394         t->n_actions = params->n_actions;
8395         t->default_action_is_const = params->default_action_is_const;
8396         t->action_data_size_max = action_data_size_max;
8397
8398         t->size = size;
8399         t->id = p->n_tables;
8400
8401         /* Node add to tailq. */
8402         TAILQ_INSERT_TAIL(&p->tables, t, node);
8403         p->n_tables++;
8404
8405         return 0;
8406 }
8407
8408 static struct rte_swx_table_params *
8409 table_params_get(struct table *table)
8410 {
8411         struct rte_swx_table_params *params;
8412         struct field *first, *last;
8413         uint8_t *key_mask;
8414         uint32_t key_size, key_offset, action_data_size, i;
8415
8416         /* Memory allocation. */
8417         params = calloc(1, sizeof(struct rte_swx_table_params));
8418         if (!params)
8419                 return NULL;
8420
8421         /* Find first (smallest offset) and last (biggest offset) match fields. */
8422         first = table->fields[0].field;
8423         last = table->fields[0].field;
8424
8425         for (i = 0; i < table->n_fields; i++) {
8426                 struct field *f = table->fields[i].field;
8427
8428                 if (f->offset < first->offset)
8429                         first = f;
8430
8431                 if (f->offset > last->offset)
8432                         last = f;
8433         }
8434
8435         /* Key offset and size. */
8436         key_offset = first->offset / 8;
8437         key_size = (last->offset + last->n_bits - first->offset) / 8;
8438
8439         /* Memory allocation. */
8440         key_mask = calloc(1, key_size);
8441         if (!key_mask) {
8442                 free(params);
8443                 return NULL;
8444         }
8445
8446         /* Key mask. */
8447         for (i = 0; i < table->n_fields; i++) {
8448                 struct field *f = table->fields[i].field;
8449                 uint32_t start = (f->offset - first->offset) / 8;
8450                 size_t size = f->n_bits / 8;
8451
8452                 memset(&key_mask[start], 0xFF, size);
8453         }
8454
8455         /* Action data size. */
8456         action_data_size = 0;
8457         for (i = 0; i < table->n_actions; i++) {
8458                 struct action *action = table->actions[i];
8459                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8460
8461                 if (ads > action_data_size)
8462                         action_data_size = ads;
8463         }
8464
8465         /* Fill in. */
8466         params->match_type = table->type->match_type;
8467         params->key_size = key_size;
8468         params->key_offset = key_offset;
8469         params->key_mask0 = key_mask;
8470         params->action_data_size = action_data_size;
8471         params->n_keys_max = table->size;
8472
8473         return params;
8474 }
8475
8476 static void
8477 table_params_free(struct rte_swx_table_params *params)
8478 {
8479         if (!params)
8480                 return;
8481
8482         free(params->key_mask0);
8483         free(params);
8484 }
8485
8486 static int
8487 table_stub_lkp(void *table __rte_unused,
8488                void *mailbox __rte_unused,
8489                uint8_t **key __rte_unused,
8490                uint64_t *action_id __rte_unused,
8491                uint8_t **action_data __rte_unused,
8492                int *hit)
8493 {
8494         *hit = 0;
8495         return 1; /* DONE. */
8496 }
8497
8498 static int
8499 table_build(struct rte_swx_pipeline *p)
8500 {
8501         uint32_t i;
8502
8503         /* Per pipeline: table statistics. */
8504         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8505         CHECK(p->table_stats, ENOMEM);
8506
8507         for (i = 0; i < p->n_tables; i++) {
8508                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8509                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8510         }
8511
8512         /* Per thread: table runt-time. */
8513         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8514                 struct thread *t = &p->threads[i];
8515                 struct table *table;
8516
8517                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8518                 CHECK(t->tables, ENOMEM);
8519
8520                 TAILQ_FOREACH(table, &p->tables, node) {
8521                         struct table_runtime *r = &t->tables[table->id];
8522
8523                         if (table->type) {
8524                                 uint64_t size;
8525
8526                                 size = table->type->ops.mailbox_size_get();
8527
8528                                 /* r->func. */
8529                                 r->func = table->type->ops.lkp;
8530
8531                                 /* r->mailbox. */
8532                                 if (size) {
8533                                         r->mailbox = calloc(1, size);
8534                                         CHECK(r->mailbox, ENOMEM);
8535                                 }
8536
8537                                 /* r->key. */
8538                                 r->key = table->header ?
8539                                         &t->structs[table->header->struct_id] :
8540                                         &t->structs[p->metadata_struct_id];
8541                         } else {
8542                                 r->func = table_stub_lkp;
8543                         }
8544                 }
8545         }
8546
8547         return 0;
8548 }
8549
8550 static void
8551 table_build_free(struct rte_swx_pipeline *p)
8552 {
8553         uint32_t i;
8554
8555         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8556                 struct thread *t = &p->threads[i];
8557                 uint32_t j;
8558
8559                 if (!t->tables)
8560                         continue;
8561
8562                 for (j = 0; j < p->n_tables; j++) {
8563                         struct table_runtime *r = &t->tables[j];
8564
8565                         free(r->mailbox);
8566                 }
8567
8568                 free(t->tables);
8569                 t->tables = NULL;
8570         }
8571
8572         if (p->table_stats) {
8573                 for (i = 0; i < p->n_tables; i++)
8574                         free(p->table_stats[i].n_pkts_action);
8575
8576                 free(p->table_stats);
8577         }
8578 }
8579
8580 static void
8581 table_free(struct rte_swx_pipeline *p)
8582 {
8583         table_build_free(p);
8584
8585         /* Tables. */
8586         for ( ; ; ) {
8587                 struct table *elem;
8588
8589                 elem = TAILQ_FIRST(&p->tables);
8590                 if (!elem)
8591                         break;
8592
8593                 TAILQ_REMOVE(&p->tables, elem, node);
8594                 free(elem->fields);
8595                 free(elem->actions);
8596                 free(elem->default_action_data);
8597                 free(elem);
8598         }
8599
8600         /* Table types. */
8601         for ( ; ; ) {
8602                 struct table_type *elem;
8603
8604                 elem = TAILQ_FIRST(&p->table_types);
8605                 if (!elem)
8606                         break;
8607
8608                 TAILQ_REMOVE(&p->table_types, elem, node);
8609                 free(elem);
8610         }
8611 }
8612
8613 /*
8614  * Selector.
8615  */
8616 static struct selector *
8617 selector_find(struct rte_swx_pipeline *p, const char *name)
8618 {
8619         struct selector *s;
8620
8621         TAILQ_FOREACH(s, &p->selectors, node)
8622                 if (strcmp(s->name, name) == 0)
8623                         return s;
8624
8625         return NULL;
8626 }
8627
8628 static struct selector *
8629 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8630 {
8631         struct selector *s = NULL;
8632
8633         TAILQ_FOREACH(s, &p->selectors, node)
8634                 if (s->id == id)
8635                         return s;
8636
8637         return NULL;
8638 }
8639
8640 static int
8641 selector_fields_check(struct rte_swx_pipeline *p,
8642                       struct rte_swx_pipeline_selector_params *params,
8643                       struct header **header)
8644 {
8645         struct header *h0 = NULL;
8646         struct field *hf, *mf;
8647         uint32_t i;
8648
8649         /* Return if no selector fields. */
8650         if (!params->n_selector_fields || !params->selector_field_names)
8651                 return -EINVAL;
8652
8653         /* Check that all the selector fields either belong to the same header
8654          * or are all meta-data fields.
8655          */
8656         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8657         mf = metadata_field_parse(p, params->selector_field_names[0]);
8658         if (!hf && !mf)
8659                 return -EINVAL;
8660
8661         for (i = 1; i < params->n_selector_fields; i++)
8662                 if (h0) {
8663                         struct header *h;
8664
8665                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8666                         if (!hf || (h->id != h0->id))
8667                                 return -EINVAL;
8668                 } else {
8669                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8670                         if (!mf)
8671                                 return -EINVAL;
8672                 }
8673
8674         /* Check that there are no duplicated match fields. */
8675         for (i = 0; i < params->n_selector_fields; i++) {
8676                 const char *field_name = params->selector_field_names[i];
8677                 uint32_t j;
8678
8679                 for (j = i + 1; j < params->n_selector_fields; j++)
8680                         if (!strcmp(params->selector_field_names[j], field_name))
8681                                 return -EINVAL;
8682         }
8683
8684         /* Return. */
8685         if (header)
8686                 *header = h0;
8687
8688         return 0;
8689 }
8690
8691 int
8692 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8693                                  const char *name,
8694                                  struct rte_swx_pipeline_selector_params *params)
8695 {
8696         struct selector *s;
8697         struct header *selector_header = NULL;
8698         struct field *group_id_field, *member_id_field;
8699         uint32_t i;
8700         int status = 0;
8701
8702         CHECK(p, EINVAL);
8703
8704         CHECK_NAME(name, EINVAL);
8705         CHECK(!table_find(p, name), EEXIST);
8706         CHECK(!selector_find(p, name), EEXIST);
8707         CHECK(!learner_find(p, name), EEXIST);
8708
8709         CHECK(params, EINVAL);
8710
8711         CHECK_NAME(params->group_id_field_name, EINVAL);
8712         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8713         CHECK(group_id_field, EINVAL);
8714
8715         for (i = 0; i < params->n_selector_fields; i++) {
8716                 const char *field_name = params->selector_field_names[i];
8717
8718                 CHECK_NAME(field_name, EINVAL);
8719         }
8720         status = selector_fields_check(p, params, &selector_header);
8721         if (status)
8722                 return status;
8723
8724         CHECK_NAME(params->member_id_field_name, EINVAL);
8725         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8726         CHECK(member_id_field, EINVAL);
8727
8728         CHECK(params->n_groups_max, EINVAL);
8729
8730         CHECK(params->n_members_per_group_max, EINVAL);
8731
8732         /* Memory allocation. */
8733         s = calloc(1, sizeof(struct selector));
8734         if (!s) {
8735                 status = -ENOMEM;
8736                 goto error;
8737         }
8738
8739         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8740         if (!s->selector_fields) {
8741                 status = -ENOMEM;
8742                 goto error;
8743         }
8744
8745         /* Node initialization. */
8746         strcpy(s->name, name);
8747
8748         s->group_id_field = group_id_field;
8749
8750         for (i = 0; i < params->n_selector_fields; i++) {
8751                 const char *field_name = params->selector_field_names[i];
8752
8753                 s->selector_fields[i] = selector_header ?
8754                         header_field_parse(p, field_name, NULL) :
8755                         metadata_field_parse(p, field_name);
8756         }
8757
8758         s->n_selector_fields = params->n_selector_fields;
8759
8760         s->selector_header = selector_header;
8761
8762         s->member_id_field = member_id_field;
8763
8764         s->n_groups_max = params->n_groups_max;
8765
8766         s->n_members_per_group_max = params->n_members_per_group_max;
8767
8768         s->id = p->n_selectors;
8769
8770         /* Node add to tailq. */
8771         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8772         p->n_selectors++;
8773
8774         return 0;
8775
8776 error:
8777         if (!s)
8778                 return status;
8779
8780         free(s->selector_fields);
8781
8782         free(s);
8783
8784         return status;
8785 }
8786
8787 static void
8788 selector_params_free(struct rte_swx_table_selector_params *params)
8789 {
8790         if (!params)
8791                 return;
8792
8793         free(params->selector_mask);
8794
8795         free(params);
8796 }
8797
8798 static struct rte_swx_table_selector_params *
8799 selector_table_params_get(struct selector *s)
8800 {
8801         struct rte_swx_table_selector_params *params = NULL;
8802         struct field *first, *last;
8803         uint32_t i;
8804
8805         /* Memory allocation. */
8806         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8807         if (!params)
8808                 goto error;
8809
8810         /* Group ID. */
8811         params->group_id_offset = s->group_id_field->offset / 8;
8812
8813         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8814         first = s->selector_fields[0];
8815         last = s->selector_fields[0];
8816
8817         for (i = 0; i < s->n_selector_fields; i++) {
8818                 struct field *f = s->selector_fields[i];
8819
8820                 if (f->offset < first->offset)
8821                         first = f;
8822
8823                 if (f->offset > last->offset)
8824                         last = f;
8825         }
8826
8827         /* Selector offset and size. */
8828         params->selector_offset = first->offset / 8;
8829         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8830
8831         /* Memory allocation. */
8832         params->selector_mask = calloc(1, params->selector_size);
8833         if (!params->selector_mask)
8834                 goto error;
8835
8836         /* Selector mask. */
8837         for (i = 0; i < s->n_selector_fields; i++) {
8838                 struct field *f = s->selector_fields[i];
8839                 uint32_t start = (f->offset - first->offset) / 8;
8840                 size_t size = f->n_bits / 8;
8841
8842                 memset(&params->selector_mask[start], 0xFF, size);
8843         }
8844
8845         /* Member ID. */
8846         params->member_id_offset = s->member_id_field->offset / 8;
8847
8848         /* Maximum number of groups. */
8849         params->n_groups_max = s->n_groups_max;
8850
8851         /* Maximum number of members per group. */
8852         params->n_members_per_group_max = s->n_members_per_group_max;
8853
8854         return params;
8855
8856 error:
8857         selector_params_free(params);
8858         return NULL;
8859 }
8860
8861 static void
8862 selector_build_free(struct rte_swx_pipeline *p)
8863 {
8864         uint32_t i;
8865
8866         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8867                 struct thread *t = &p->threads[i];
8868                 uint32_t j;
8869
8870                 if (!t->selectors)
8871                         continue;
8872
8873                 for (j = 0; j < p->n_selectors; j++) {
8874                         struct selector_runtime *r = &t->selectors[j];
8875
8876                         free(r->mailbox);
8877                 }
8878
8879                 free(t->selectors);
8880                 t->selectors = NULL;
8881         }
8882
8883         free(p->selector_stats);
8884         p->selector_stats = NULL;
8885 }
8886
8887 static int
8888 selector_build(struct rte_swx_pipeline *p)
8889 {
8890         uint32_t i;
8891         int status = 0;
8892
8893         /* Per pipeline: selector statistics. */
8894         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8895         if (!p->selector_stats) {
8896                 status = -ENOMEM;
8897                 goto error;
8898         }
8899
8900         /* Per thread: selector run-time. */
8901         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8902                 struct thread *t = &p->threads[i];
8903                 struct selector *s;
8904
8905                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8906                 if (!t->selectors) {
8907                         status = -ENOMEM;
8908                         goto error;
8909                 }
8910
8911                 TAILQ_FOREACH(s, &p->selectors, node) {
8912                         struct selector_runtime *r = &t->selectors[s->id];
8913                         uint64_t size;
8914
8915                         /* r->mailbox. */
8916                         size = rte_swx_table_selector_mailbox_size_get();
8917                         if (size) {
8918                                 r->mailbox = calloc(1, size);
8919                                 if (!r->mailbox) {
8920                                         status = -ENOMEM;
8921                                         goto error;
8922                                 }
8923                         }
8924
8925                         /* r->group_id_buffer. */
8926                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8927
8928                         /* r->selector_buffer. */
8929                         r->selector_buffer = s->selector_header ?
8930                                 &t->structs[s->selector_header->struct_id] :
8931                                 &t->structs[p->metadata_struct_id];
8932
8933                         /* r->member_id_buffer. */
8934                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8935                 }
8936         }
8937
8938         return 0;
8939
8940 error:
8941         selector_build_free(p);
8942         return status;
8943 }
8944
8945 static void
8946 selector_free(struct rte_swx_pipeline *p)
8947 {
8948         selector_build_free(p);
8949
8950         /* Selector tables. */
8951         for ( ; ; ) {
8952                 struct selector *elem;
8953
8954                 elem = TAILQ_FIRST(&p->selectors);
8955                 if (!elem)
8956                         break;
8957
8958                 TAILQ_REMOVE(&p->selectors, elem, node);
8959                 free(elem->selector_fields);
8960                 free(elem);
8961         }
8962 }
8963
8964 /*
8965  * Learner table.
8966  */
8967 static struct learner *
8968 learner_find(struct rte_swx_pipeline *p, const char *name)
8969 {
8970         struct learner *l;
8971
8972         TAILQ_FOREACH(l, &p->learners, node)
8973                 if (!strcmp(l->name, name))
8974                         return l;
8975
8976         return NULL;
8977 }
8978
8979 static struct learner *
8980 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8981 {
8982         struct learner *l = NULL;
8983
8984         TAILQ_FOREACH(l, &p->learners, node)
8985                 if (l->id == id)
8986                         return l;
8987
8988         return NULL;
8989 }
8990
8991 static int
8992 learner_match_fields_check(struct rte_swx_pipeline *p,
8993                            struct rte_swx_pipeline_learner_params *params,
8994                            struct header **header)
8995 {
8996         struct header *h0 = NULL;
8997         struct field *hf, *mf;
8998         uint32_t i;
8999
9000         /* Return if no match fields. */
9001         if (!params->n_fields || !params->field_names)
9002                 return -EINVAL;
9003
9004         /* Check that all the match fields either belong to the same header
9005          * or are all meta-data fields.
9006          */
9007         hf = header_field_parse(p, params->field_names[0], &h0);
9008         mf = metadata_field_parse(p, params->field_names[0]);
9009         if (!hf && !mf)
9010                 return -EINVAL;
9011
9012         for (i = 1; i < params->n_fields; i++)
9013                 if (h0) {
9014                         struct header *h;
9015
9016                         hf = header_field_parse(p, params->field_names[i], &h);
9017                         if (!hf || (h->id != h0->id))
9018                                 return -EINVAL;
9019                 } else {
9020                         mf = metadata_field_parse(p, params->field_names[i]);
9021                         if (!mf)
9022                                 return -EINVAL;
9023                 }
9024
9025         /* Check that there are no duplicated match fields. */
9026         for (i = 0; i < params->n_fields; i++) {
9027                 const char *field_name = params->field_names[i];
9028                 uint32_t j;
9029
9030                 for (j = i + 1; j < params->n_fields; j++)
9031                         if (!strcmp(params->field_names[j], field_name))
9032                                 return -EINVAL;
9033         }
9034
9035         /* Return. */
9036         if (header)
9037                 *header = h0;
9038
9039         return 0;
9040 }
9041
9042 static int
9043 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
9044 {
9045         struct struct_type *mst = p->metadata_st, *ast = a->st;
9046         struct field *mf, *af;
9047         uint32_t mf_pos, i;
9048
9049         if (!ast) {
9050                 if (mf_name)
9051                         return -EINVAL;
9052
9053                 return 0;
9054         }
9055
9056         /* Check that mf_name is the name of a valid meta-data field. */
9057         CHECK_NAME(mf_name, EINVAL);
9058         mf = metadata_field_parse(p, mf_name);
9059         CHECK(mf, EINVAL);
9060
9061         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
9062          * all the action arguments.
9063          */
9064         mf_pos = mf - mst->fields;
9065         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
9066
9067         /* Check that the size of each of the identified meta-data fields matches exactly the size
9068          * of the corresponding action argument.
9069          */
9070         for (i = 0; i < ast->n_fields; i++) {
9071                 mf = &mst->fields[mf_pos + i];
9072                 af = &ast->fields[i];
9073
9074                 CHECK(mf->n_bits == af->n_bits, EINVAL);
9075         }
9076
9077         return 0;
9078 }
9079
9080 static int
9081 learner_action_learning_check(struct rte_swx_pipeline *p,
9082                               struct action *action,
9083                               const char **action_names,
9084                               uint32_t n_actions)
9085 {
9086         uint32_t i;
9087
9088         /* For each "learn" instruction of the current action, check that the learned action (i.e.
9089          * the action passed as argument to the "learn" instruction) is also enabled for the
9090          * current learner table.
9091          */
9092         for (i = 0; i < action->n_instructions; i++) {
9093                 struct instruction *instr = &action->instructions[i];
9094                 uint32_t found = 0, j;
9095
9096                 if (instr->type != INSTR_LEARNER_LEARN)
9097                         continue;
9098
9099                 for (j = 0; j < n_actions; j++) {
9100                         struct action *a;
9101
9102                         a = action_find(p, action_names[j]);
9103                         if (!a)
9104                                 return -EINVAL;
9105
9106                         if (a->id == instr->learn.action_id)
9107                                 found = 1;
9108                 }
9109
9110                 if (!found)
9111                         return -EINVAL;
9112         }
9113
9114         return 0;
9115 }
9116
9117 int
9118 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
9119                               const char *name,
9120                               struct rte_swx_pipeline_learner_params *params,
9121                               uint32_t size,
9122                               uint32_t timeout)
9123 {
9124         struct learner *l = NULL;
9125         struct action *default_action;
9126         struct header *header = NULL;
9127         uint32_t action_data_size_max = 0, i;
9128         int status = 0;
9129
9130         CHECK(p, EINVAL);
9131
9132         CHECK_NAME(name, EINVAL);
9133         CHECK(!table_find(p, name), EEXIST);
9134         CHECK(!selector_find(p, name), EEXIST);
9135         CHECK(!learner_find(p, name), EEXIST);
9136
9137         CHECK(params, EINVAL);
9138
9139         /* Match checks. */
9140         status = learner_match_fields_check(p, params, &header);
9141         if (status)
9142                 return status;
9143
9144         /* Action checks. */
9145         CHECK(params->n_actions, EINVAL);
9146
9147         CHECK(params->action_names, EINVAL);
9148         for (i = 0; i < params->n_actions; i++) {
9149                 const char *action_name = params->action_names[i];
9150                 const char *action_field_name = params->action_field_names[i];
9151                 struct action *a;
9152                 uint32_t action_data_size;
9153
9154                 CHECK_NAME(action_name, EINVAL);
9155
9156                 a = action_find(p, action_name);
9157                 CHECK(a, EINVAL);
9158
9159                 status = learner_action_args_check(p, a, action_field_name);
9160                 if (status)
9161                         return status;
9162
9163                 status = learner_action_learning_check(p,
9164                                                        a,
9165                                                        params->action_names,
9166                                                        params->n_actions);
9167                 if (status)
9168                         return status;
9169
9170                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9171                 if (action_data_size > action_data_size_max)
9172                         action_data_size_max = action_data_size;
9173         }
9174
9175         CHECK_NAME(params->default_action_name, EINVAL);
9176         for (i = 0; i < p->n_actions; i++)
9177                 if (!strcmp(params->action_names[i],
9178                             params->default_action_name))
9179                         break;
9180         CHECK(i < params->n_actions, EINVAL);
9181
9182         default_action = action_find(p, params->default_action_name);
9183         CHECK((default_action->st && params->default_action_data) ||
9184               !params->default_action_data, EINVAL);
9185
9186         /* Any other checks. */
9187         CHECK(size, EINVAL);
9188         CHECK(timeout, EINVAL);
9189
9190         /* Memory allocation. */
9191         l = calloc(1, sizeof(struct learner));
9192         if (!l)
9193                 goto nomem;
9194
9195         l->fields = calloc(params->n_fields, sizeof(struct field *));
9196         if (!l->fields)
9197                 goto nomem;
9198
9199         l->actions = calloc(params->n_actions, sizeof(struct action *));
9200         if (!l->actions)
9201                 goto nomem;
9202
9203         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9204         if (!l->action_arg)
9205                 goto nomem;
9206
9207         if (action_data_size_max) {
9208                 l->default_action_data = calloc(1, action_data_size_max);
9209                 if (!l->default_action_data)
9210                         goto nomem;
9211         }
9212
9213         /* Node initialization. */
9214         strcpy(l->name, name);
9215
9216         for (i = 0; i < params->n_fields; i++) {
9217                 const char *field_name = params->field_names[i];
9218
9219                 l->fields[i] = header ?
9220                         header_field_parse(p, field_name, NULL) :
9221                         metadata_field_parse(p, field_name);
9222         }
9223
9224         l->n_fields = params->n_fields;
9225
9226         l->header = header;
9227
9228         for (i = 0; i < params->n_actions; i++) {
9229                 const char *mf_name = params->action_field_names[i];
9230
9231                 l->actions[i] = action_find(p, params->action_names[i]);
9232
9233                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9234         }
9235
9236         l->default_action = default_action;
9237
9238         if (default_action->st)
9239                 memcpy(l->default_action_data,
9240                        params->default_action_data,
9241                        default_action->st->n_bits / 8);
9242
9243         l->n_actions = params->n_actions;
9244
9245         l->default_action_is_const = params->default_action_is_const;
9246
9247         l->action_data_size_max = action_data_size_max;
9248
9249         l->size = size;
9250
9251         l->timeout = timeout;
9252
9253         l->id = p->n_learners;
9254
9255         /* Node add to tailq. */
9256         TAILQ_INSERT_TAIL(&p->learners, l, node);
9257         p->n_learners++;
9258
9259         return 0;
9260
9261 nomem:
9262         if (!l)
9263                 return -ENOMEM;
9264
9265         free(l->action_arg);
9266         free(l->actions);
9267         free(l->fields);
9268         free(l);
9269
9270         return -ENOMEM;
9271 }
9272
9273 static void
9274 learner_params_free(struct rte_swx_table_learner_params *params)
9275 {
9276         if (!params)
9277                 return;
9278
9279         free(params->key_mask0);
9280
9281         free(params);
9282 }
9283
9284 static struct rte_swx_table_learner_params *
9285 learner_params_get(struct learner *l)
9286 {
9287         struct rte_swx_table_learner_params *params = NULL;
9288         struct field *first, *last;
9289         uint32_t i;
9290
9291         /* Memory allocation. */
9292         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9293         if (!params)
9294                 goto error;
9295
9296         /* Find first (smallest offset) and last (biggest offset) match fields. */
9297         first = l->fields[0];
9298         last = l->fields[0];
9299
9300         for (i = 0; i < l->n_fields; i++) {
9301                 struct field *f = l->fields[i];
9302
9303                 if (f->offset < first->offset)
9304                         first = f;
9305
9306                 if (f->offset > last->offset)
9307                         last = f;
9308         }
9309
9310         /* Key offset and size. */
9311         params->key_offset = first->offset / 8;
9312         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9313
9314         /* Memory allocation. */
9315         params->key_mask0 = calloc(1, params->key_size);
9316         if (!params->key_mask0)
9317                 goto error;
9318
9319         /* Key mask. */
9320         for (i = 0; i < l->n_fields; i++) {
9321                 struct field *f = l->fields[i];
9322                 uint32_t start = (f->offset - first->offset) / 8;
9323                 size_t size = f->n_bits / 8;
9324
9325                 memset(&params->key_mask0[start], 0xFF, size);
9326         }
9327
9328         /* Action data size. */
9329         params->action_data_size = l->action_data_size_max;
9330
9331         /* Maximum number of keys. */
9332         params->n_keys_max = l->size;
9333
9334         /* Timeout. */
9335         params->key_timeout = l->timeout;
9336
9337         return params;
9338
9339 error:
9340         learner_params_free(params);
9341         return NULL;
9342 }
9343
9344 static void
9345 learner_build_free(struct rte_swx_pipeline *p)
9346 {
9347         uint32_t i;
9348
9349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9350                 struct thread *t = &p->threads[i];
9351                 uint32_t j;
9352
9353                 if (!t->learners)
9354                         continue;
9355
9356                 for (j = 0; j < p->n_learners; j++) {
9357                         struct learner_runtime *r = &t->learners[j];
9358
9359                         free(r->mailbox);
9360                         free(r->action_data);
9361                 }
9362
9363                 free(t->learners);
9364                 t->learners = NULL;
9365         }
9366
9367         if (p->learner_stats) {
9368                 for (i = 0; i < p->n_learners; i++)
9369                         free(p->learner_stats[i].n_pkts_action);
9370
9371                 free(p->learner_stats);
9372         }
9373 }
9374
9375 static int
9376 learner_build(struct rte_swx_pipeline *p)
9377 {
9378         uint32_t i;
9379         int status = 0;
9380
9381         /* Per pipeline: learner statistics. */
9382         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9383         CHECK(p->learner_stats, ENOMEM);
9384
9385         for (i = 0; i < p->n_learners; i++) {
9386                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9387                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9388         }
9389
9390         /* Per thread: learner run-time. */
9391         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9392                 struct thread *t = &p->threads[i];
9393                 struct learner *l;
9394
9395                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9396                 if (!t->learners) {
9397                         status = -ENOMEM;
9398                         goto error;
9399                 }
9400
9401                 TAILQ_FOREACH(l, &p->learners, node) {
9402                         struct learner_runtime *r = &t->learners[l->id];
9403                         uint64_t size;
9404                         uint32_t j;
9405
9406                         /* r->mailbox. */
9407                         size = rte_swx_table_learner_mailbox_size_get();
9408                         if (size) {
9409                                 r->mailbox = calloc(1, size);
9410                                 if (!r->mailbox) {
9411                                         status = -ENOMEM;
9412                                         goto error;
9413                                 }
9414                         }
9415
9416                         /* r->key. */
9417                         r->key = l->header ?
9418                                 &t->structs[l->header->struct_id] :
9419                                 &t->structs[p->metadata_struct_id];
9420
9421                         /* r->action_data. */
9422                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9423                         if (!r->action_data) {
9424                                 status = -ENOMEM;
9425                                 goto error;
9426                         }
9427
9428                         for (j = 0; j < l->n_actions; j++) {
9429                                 struct action *a = l->actions[j];
9430                                 struct field *mf = l->action_arg[j];
9431                                 uint8_t *m = t->structs[p->metadata_struct_id];
9432
9433                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9434                         }
9435                 }
9436         }
9437
9438         return 0;
9439
9440 error:
9441         learner_build_free(p);
9442         return status;
9443 }
9444
9445 static void
9446 learner_free(struct rte_swx_pipeline *p)
9447 {
9448         learner_build_free(p);
9449
9450         /* Learner tables. */
9451         for ( ; ; ) {
9452                 struct learner *l;
9453
9454                 l = TAILQ_FIRST(&p->learners);
9455                 if (!l)
9456                         break;
9457
9458                 TAILQ_REMOVE(&p->learners, l, node);
9459                 free(l->fields);
9460                 free(l->actions);
9461                 free(l->action_arg);
9462                 free(l->default_action_data);
9463                 free(l);
9464         }
9465 }
9466
9467 /*
9468  * Table state.
9469  */
9470 static int
9471 table_state_build(struct rte_swx_pipeline *p)
9472 {
9473         struct table *table;
9474         struct selector *s;
9475         struct learner *l;
9476
9477         p->table_state = calloc(p->n_tables + p->n_selectors,
9478                                 sizeof(struct rte_swx_table_state));
9479         CHECK(p->table_state, ENOMEM);
9480
9481         TAILQ_FOREACH(table, &p->tables, node) {
9482                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9483
9484                 if (table->type) {
9485                         struct rte_swx_table_params *params;
9486
9487                         /* ts->obj. */
9488                         params = table_params_get(table);
9489                         CHECK(params, ENOMEM);
9490
9491                         ts->obj = table->type->ops.create(params,
9492                                 NULL,
9493                                 table->args,
9494                                 p->numa_node);
9495
9496                         table_params_free(params);
9497                         CHECK(ts->obj, ENODEV);
9498                 }
9499
9500                 /* ts->default_action_data. */
9501                 if (table->action_data_size_max) {
9502                         ts->default_action_data =
9503                                 malloc(table->action_data_size_max);
9504                         CHECK(ts->default_action_data, ENOMEM);
9505
9506                         memcpy(ts->default_action_data,
9507                                table->default_action_data,
9508                                table->action_data_size_max);
9509                 }
9510
9511                 /* ts->default_action_id. */
9512                 ts->default_action_id = table->default_action->id;
9513         }
9514
9515         TAILQ_FOREACH(s, &p->selectors, node) {
9516                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9517                 struct rte_swx_table_selector_params *params;
9518
9519                 /* ts->obj. */
9520                 params = selector_table_params_get(s);
9521                 CHECK(params, ENOMEM);
9522
9523                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9524
9525                 selector_params_free(params);
9526                 CHECK(ts->obj, ENODEV);
9527         }
9528
9529         TAILQ_FOREACH(l, &p->learners, node) {
9530                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9531                         p->n_selectors + l->id];
9532                 struct rte_swx_table_learner_params *params;
9533
9534                 /* ts->obj. */
9535                 params = learner_params_get(l);
9536                 CHECK(params, ENOMEM);
9537
9538                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9539                 learner_params_free(params);
9540                 CHECK(ts->obj, ENODEV);
9541
9542                 /* ts->default_action_data. */
9543                 if (l->action_data_size_max) {
9544                         ts->default_action_data = malloc(l->action_data_size_max);
9545                         CHECK(ts->default_action_data, ENOMEM);
9546
9547                         memcpy(ts->default_action_data,
9548                                l->default_action_data,
9549                                l->action_data_size_max);
9550                 }
9551
9552                 /* ts->default_action_id. */
9553                 ts->default_action_id = l->default_action->id;
9554         }
9555
9556         return 0;
9557 }
9558
9559 static void
9560 table_state_build_free(struct rte_swx_pipeline *p)
9561 {
9562         uint32_t i;
9563
9564         if (!p->table_state)
9565                 return;
9566
9567         for (i = 0; i < p->n_tables; i++) {
9568                 struct rte_swx_table_state *ts = &p->table_state[i];
9569                 struct table *table = table_find_by_id(p, i);
9570
9571                 /* ts->obj. */
9572                 if (table->type && ts->obj)
9573                         table->type->ops.free(ts->obj);
9574
9575                 /* ts->default_action_data. */
9576                 free(ts->default_action_data);
9577         }
9578
9579         for (i = 0; i < p->n_selectors; i++) {
9580                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9581
9582                 /* ts->obj. */
9583                 if (ts->obj)
9584                         rte_swx_table_selector_free(ts->obj);
9585         }
9586
9587         for (i = 0; i < p->n_learners; i++) {
9588                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9589
9590                 /* ts->obj. */
9591                 if (ts->obj)
9592                         rte_swx_table_learner_free(ts->obj);
9593
9594                 /* ts->default_action_data. */
9595                 free(ts->default_action_data);
9596         }
9597
9598         free(p->table_state);
9599         p->table_state = NULL;
9600 }
9601
9602 static void
9603 table_state_free(struct rte_swx_pipeline *p)
9604 {
9605         table_state_build_free(p);
9606 }
9607
9608 /*
9609  * Register array.
9610  */
9611 static struct regarray *
9612 regarray_find(struct rte_swx_pipeline *p, const char *name)
9613 {
9614         struct regarray *elem;
9615
9616         TAILQ_FOREACH(elem, &p->regarrays, node)
9617                 if (!strcmp(elem->name, name))
9618                         return elem;
9619
9620         return NULL;
9621 }
9622
9623 static struct regarray *
9624 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9625 {
9626         struct regarray *elem = NULL;
9627
9628         TAILQ_FOREACH(elem, &p->regarrays, node)
9629                 if (elem->id == id)
9630                         return elem;
9631
9632         return NULL;
9633 }
9634
9635 int
9636 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9637                               const char *name,
9638                               uint32_t size,
9639                               uint64_t init_val)
9640 {
9641         struct regarray *r;
9642
9643         CHECK(p, EINVAL);
9644
9645         CHECK_NAME(name, EINVAL);
9646         CHECK(!regarray_find(p, name), EEXIST);
9647
9648         CHECK(size, EINVAL);
9649         size = rte_align32pow2(size);
9650
9651         /* Memory allocation. */
9652         r = calloc(1, sizeof(struct regarray));
9653         CHECK(r, ENOMEM);
9654
9655         /* Node initialization. */
9656         strcpy(r->name, name);
9657         r->init_val = init_val;
9658         r->size = size;
9659         r->id = p->n_regarrays;
9660
9661         /* Node add to tailq. */
9662         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9663         p->n_regarrays++;
9664
9665         return 0;
9666 }
9667
9668 static int
9669 regarray_build(struct rte_swx_pipeline *p)
9670 {
9671         struct regarray *regarray;
9672
9673         if (!p->n_regarrays)
9674                 return 0;
9675
9676         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9677         CHECK(p->regarray_runtime, ENOMEM);
9678
9679         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9680                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9681                 uint32_t i;
9682
9683                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9684                                          RTE_CACHE_LINE_SIZE,
9685                                          p->numa_node);
9686                 CHECK(r->regarray, ENOMEM);
9687
9688                 if (regarray->init_val)
9689                         for (i = 0; i < regarray->size; i++)
9690                                 r->regarray[i] = regarray->init_val;
9691
9692                 r->size_mask = regarray->size - 1;
9693         }
9694
9695         return 0;
9696 }
9697
9698 static void
9699 regarray_build_free(struct rte_swx_pipeline *p)
9700 {
9701         uint32_t i;
9702
9703         if (!p->regarray_runtime)
9704                 return;
9705
9706         for (i = 0; i < p->n_regarrays; i++) {
9707                 struct regarray *regarray = regarray_find_by_id(p, i);
9708                 struct regarray_runtime *r = &p->regarray_runtime[i];
9709
9710                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9711         }
9712
9713         free(p->regarray_runtime);
9714         p->regarray_runtime = NULL;
9715 }
9716
9717 static void
9718 regarray_free(struct rte_swx_pipeline *p)
9719 {
9720         regarray_build_free(p);
9721
9722         for ( ; ; ) {
9723                 struct regarray *elem;
9724
9725                 elem = TAILQ_FIRST(&p->regarrays);
9726                 if (!elem)
9727                         break;
9728
9729                 TAILQ_REMOVE(&p->regarrays, elem, node);
9730                 free(elem);
9731         }
9732 }
9733
9734 /*
9735  * Meter array.
9736  */
9737 static struct meter_profile *
9738 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9739 {
9740         struct meter_profile *elem;
9741
9742         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9743                 if (!strcmp(elem->name, name))
9744                         return elem;
9745
9746         return NULL;
9747 }
9748
9749 static struct metarray *
9750 metarray_find(struct rte_swx_pipeline *p, const char *name)
9751 {
9752         struct metarray *elem;
9753
9754         TAILQ_FOREACH(elem, &p->metarrays, node)
9755                 if (!strcmp(elem->name, name))
9756                         return elem;
9757
9758         return NULL;
9759 }
9760
9761 static struct metarray *
9762 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9763 {
9764         struct metarray *elem = NULL;
9765
9766         TAILQ_FOREACH(elem, &p->metarrays, node)
9767                 if (elem->id == id)
9768                         return elem;
9769
9770         return NULL;
9771 }
9772
9773 int
9774 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9775                                  const char *name,
9776                                  uint32_t size)
9777 {
9778         struct metarray *m;
9779
9780         CHECK(p, EINVAL);
9781
9782         CHECK_NAME(name, EINVAL);
9783         CHECK(!metarray_find(p, name), EEXIST);
9784
9785         CHECK(size, EINVAL);
9786         size = rte_align32pow2(size);
9787
9788         /* Memory allocation. */
9789         m = calloc(1, sizeof(struct metarray));
9790         CHECK(m, ENOMEM);
9791
9792         /* Node initialization. */
9793         strcpy(m->name, name);
9794         m->size = size;
9795         m->id = p->n_metarrays;
9796
9797         /* Node add to tailq. */
9798         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9799         p->n_metarrays++;
9800
9801         return 0;
9802 }
9803
9804 struct meter_profile meter_profile_default = {
9805         .node = {0},
9806         .name = "",
9807         .params = {0},
9808
9809         .profile = {
9810                 .cbs = 10000,
9811                 .pbs = 10000,
9812                 .cir_period = 1,
9813                 .cir_bytes_per_period = 1,
9814                 .pir_period = 1,
9815                 .pir_bytes_per_period = 1,
9816         },
9817
9818         .n_users = 0,
9819 };
9820
9821 static void
9822 meter_init(struct meter *m)
9823 {
9824         memset(m, 0, sizeof(struct meter));
9825         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9826         m->profile = &meter_profile_default;
9827         m->color_mask = RTE_COLOR_GREEN;
9828
9829         meter_profile_default.n_users++;
9830 }
9831
9832 static int
9833 metarray_build(struct rte_swx_pipeline *p)
9834 {
9835         struct metarray *m;
9836
9837         if (!p->n_metarrays)
9838                 return 0;
9839
9840         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9841         CHECK(p->metarray_runtime, ENOMEM);
9842
9843         TAILQ_FOREACH(m, &p->metarrays, node) {
9844                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9845                 uint32_t i;
9846
9847                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9848                                          RTE_CACHE_LINE_SIZE,
9849                                          p->numa_node);
9850                 CHECK(r->metarray, ENOMEM);
9851
9852                 for (i = 0; i < m->size; i++)
9853                         meter_init(&r->metarray[i]);
9854
9855                 r->size_mask = m->size - 1;
9856         }
9857
9858         return 0;
9859 }
9860
9861 static void
9862 metarray_build_free(struct rte_swx_pipeline *p)
9863 {
9864         uint32_t i;
9865
9866         if (!p->metarray_runtime)
9867                 return;
9868
9869         for (i = 0; i < p->n_metarrays; i++) {
9870                 struct metarray *m = metarray_find_by_id(p, i);
9871                 struct metarray_runtime *r = &p->metarray_runtime[i];
9872
9873                 env_free(r->metarray, m->size * sizeof(struct meter));
9874         }
9875
9876         free(p->metarray_runtime);
9877         p->metarray_runtime = NULL;
9878 }
9879
9880 static void
9881 metarray_free(struct rte_swx_pipeline *p)
9882 {
9883         metarray_build_free(p);
9884
9885         /* Meter arrays. */
9886         for ( ; ; ) {
9887                 struct metarray *elem;
9888
9889                 elem = TAILQ_FIRST(&p->metarrays);
9890                 if (!elem)
9891                         break;
9892
9893                 TAILQ_REMOVE(&p->metarrays, elem, node);
9894                 free(elem);
9895         }
9896
9897         /* Meter profiles. */
9898         for ( ; ; ) {
9899                 struct meter_profile *elem;
9900
9901                 elem = TAILQ_FIRST(&p->meter_profiles);
9902                 if (!elem)
9903                         break;
9904
9905                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9906                 free(elem);
9907         }
9908 }
9909
9910 /*
9911  * Pipeline.
9912  */
9913 int
9914 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9915 {
9916         struct rte_swx_pipeline *pipeline;
9917
9918         /* Check input parameters. */
9919         CHECK(p, EINVAL);
9920
9921         /* Memory allocation. */
9922         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9923         CHECK(pipeline, ENOMEM);
9924
9925         /* Initialization. */
9926         TAILQ_INIT(&pipeline->struct_types);
9927         TAILQ_INIT(&pipeline->port_in_types);
9928         TAILQ_INIT(&pipeline->ports_in);
9929         TAILQ_INIT(&pipeline->port_out_types);
9930         TAILQ_INIT(&pipeline->ports_out);
9931         TAILQ_INIT(&pipeline->extern_types);
9932         TAILQ_INIT(&pipeline->extern_objs);
9933         TAILQ_INIT(&pipeline->extern_funcs);
9934         TAILQ_INIT(&pipeline->headers);
9935         TAILQ_INIT(&pipeline->actions);
9936         TAILQ_INIT(&pipeline->table_types);
9937         TAILQ_INIT(&pipeline->tables);
9938         TAILQ_INIT(&pipeline->selectors);
9939         TAILQ_INIT(&pipeline->learners);
9940         TAILQ_INIT(&pipeline->regarrays);
9941         TAILQ_INIT(&pipeline->meter_profiles);
9942         TAILQ_INIT(&pipeline->metarrays);
9943
9944         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9945         pipeline->numa_node = numa_node;
9946
9947         *p = pipeline;
9948         return 0;
9949 }
9950
9951 void
9952 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9953 {
9954         if (!p)
9955                 return;
9956
9957         free(p->instructions);
9958
9959         metarray_free(p);
9960         regarray_free(p);
9961         table_state_free(p);
9962         learner_free(p);
9963         selector_free(p);
9964         table_free(p);
9965         action_free(p);
9966         metadata_free(p);
9967         header_free(p);
9968         extern_func_free(p);
9969         extern_obj_free(p);
9970         port_out_free(p);
9971         port_in_free(p);
9972         struct_free(p);
9973
9974         free(p);
9975 }
9976
9977 int
9978 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9979                                      const char **instructions,
9980                                      uint32_t n_instructions)
9981 {
9982         int err;
9983         uint32_t i;
9984
9985         err = instruction_config(p, NULL, instructions, n_instructions);
9986         if (err)
9987                 return err;
9988
9989         /* Thread instruction pointer reset. */
9990         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9991                 struct thread *t = &p->threads[i];
9992
9993                 thread_ip_reset(p, t);
9994         }
9995
9996         return 0;
9997 }
9998
9999 int
10000 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10001 {
10002         int status;
10003
10004         CHECK(p, EINVAL);
10005         CHECK(p->build_done == 0, EEXIST);
10006
10007         status = port_in_build(p);
10008         if (status)
10009                 goto error;
10010
10011         status = port_out_build(p);
10012         if (status)
10013                 goto error;
10014
10015         status = struct_build(p);
10016         if (status)
10017                 goto error;
10018
10019         status = extern_obj_build(p);
10020         if (status)
10021                 goto error;
10022
10023         status = extern_func_build(p);
10024         if (status)
10025                 goto error;
10026
10027         status = header_build(p);
10028         if (status)
10029                 goto error;
10030
10031         status = metadata_build(p);
10032         if (status)
10033                 goto error;
10034
10035         status = action_build(p);
10036         if (status)
10037                 goto error;
10038
10039         status = table_build(p);
10040         if (status)
10041                 goto error;
10042
10043         status = selector_build(p);
10044         if (status)
10045                 goto error;
10046
10047         status = learner_build(p);
10048         if (status)
10049                 goto error;
10050
10051         status = table_state_build(p);
10052         if (status)
10053                 goto error;
10054
10055         status = regarray_build(p);
10056         if (status)
10057                 goto error;
10058
10059         status = metarray_build(p);
10060         if (status)
10061                 goto error;
10062
10063         p->build_done = 1;
10064         return 0;
10065
10066 error:
10067         metarray_build_free(p);
10068         regarray_build_free(p);
10069         table_state_build_free(p);
10070         learner_build_free(p);
10071         selector_build_free(p);
10072         table_build_free(p);
10073         action_build_free(p);
10074         metadata_build_free(p);
10075         header_build_free(p);
10076         extern_func_build_free(p);
10077         extern_obj_build_free(p);
10078         port_out_build_free(p);
10079         port_in_build_free(p);
10080         struct_build_free(p);
10081
10082         return status;
10083 }
10084
10085 void
10086 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10087 {
10088         uint32_t i;
10089
10090         for (i = 0; i < n_instructions; i++)
10091                 instr_exec(p);
10092 }
10093
10094 void
10095 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10096 {
10097         uint32_t i;
10098
10099         for (i = 0; i < p->n_ports_out; i++) {
10100                 struct port_out_runtime *port = &p->out[i];
10101
10102                 if (port->flush)
10103                         port->flush(port->obj);
10104         }
10105 }
10106
10107 /*
10108  * Control.
10109  */
10110 int
10111 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10112                               struct rte_swx_ctl_pipeline_info *pipeline)
10113 {
10114         struct action *action;
10115         struct table *table;
10116         uint32_t n_actions = 0, n_tables = 0;
10117
10118         if (!p || !pipeline)
10119                 return -EINVAL;
10120
10121         TAILQ_FOREACH(action, &p->actions, node)
10122                 n_actions++;
10123
10124         TAILQ_FOREACH(table, &p->tables, node)
10125                 n_tables++;
10126
10127         pipeline->n_ports_in = p->n_ports_in;
10128         pipeline->n_ports_out = p->n_ports_out;
10129         pipeline->n_actions = n_actions;
10130         pipeline->n_tables = n_tables;
10131         pipeline->n_selectors = p->n_selectors;
10132         pipeline->n_learners = p->n_learners;
10133         pipeline->n_regarrays = p->n_regarrays;
10134         pipeline->n_metarrays = p->n_metarrays;
10135
10136         return 0;
10137 }
10138
10139 int
10140 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10141 {
10142         if (!p || !numa_node)
10143                 return -EINVAL;
10144
10145         *numa_node = p->numa_node;
10146         return 0;
10147 }
10148
10149 int
10150 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10151                             uint32_t action_id,
10152                             struct rte_swx_ctl_action_info *action)
10153 {
10154         struct action *a = NULL;
10155
10156         if (!p || (action_id >= p->n_actions) || !action)
10157                 return -EINVAL;
10158
10159         a = action_find_by_id(p, action_id);
10160         if (!a)
10161                 return -EINVAL;
10162
10163         strcpy(action->name, a->name);
10164         action->n_args = a->st ? a->st->n_fields : 0;
10165         return 0;
10166 }
10167
10168 int
10169 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10170                                 uint32_t action_id,
10171                                 uint32_t action_arg_id,
10172                                 struct rte_swx_ctl_action_arg_info *action_arg)
10173 {
10174         struct action *a = NULL;
10175         struct field *arg = NULL;
10176
10177         if (!p || (action_id >= p->n_actions) || !action_arg)
10178                 return -EINVAL;
10179
10180         a = action_find_by_id(p, action_id);
10181         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10182                 return -EINVAL;
10183
10184         arg = &a->st->fields[action_arg_id];
10185         strcpy(action_arg->name, arg->name);
10186         action_arg->n_bits = arg->n_bits;
10187         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10188
10189         return 0;
10190 }
10191
10192 int
10193 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10194                            uint32_t table_id,
10195                            struct rte_swx_ctl_table_info *table)
10196 {
10197         struct table *t = NULL;
10198
10199         if (!p || !table)
10200                 return -EINVAL;
10201
10202         t = table_find_by_id(p, table_id);
10203         if (!t)
10204                 return -EINVAL;
10205
10206         strcpy(table->name, t->name);
10207         strcpy(table->args, t->args);
10208         table->n_match_fields = t->n_fields;
10209         table->n_actions = t->n_actions;
10210         table->default_action_is_const = t->default_action_is_const;
10211         table->size = t->size;
10212         return 0;
10213 }
10214
10215 int
10216 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10217         uint32_t table_id,
10218         uint32_t match_field_id,
10219         struct rte_swx_ctl_table_match_field_info *match_field)
10220 {
10221         struct table *t;
10222         struct match_field *f;
10223
10224         if (!p || (table_id >= p->n_tables) || !match_field)
10225                 return -EINVAL;
10226
10227         t = table_find_by_id(p, table_id);
10228         if (!t || (match_field_id >= t->n_fields))
10229                 return -EINVAL;
10230
10231         f = &t->fields[match_field_id];
10232         match_field->match_type = f->match_type;
10233         match_field->is_header = t->header ? 1 : 0;
10234         match_field->n_bits = f->field->n_bits;
10235         match_field->offset = f->field->offset;
10236
10237         return 0;
10238 }
10239
10240 int
10241 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10242         uint32_t table_id,
10243         uint32_t table_action_id,
10244         struct rte_swx_ctl_table_action_info *table_action)
10245 {
10246         struct table *t;
10247
10248         if (!p || (table_id >= p->n_tables) || !table_action)
10249                 return -EINVAL;
10250
10251         t = table_find_by_id(p, table_id);
10252         if (!t || (table_action_id >= t->n_actions))
10253                 return -EINVAL;
10254
10255         table_action->action_id = t->actions[table_action_id]->id;
10256
10257         return 0;
10258 }
10259
10260 int
10261 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10262                           uint32_t table_id,
10263                           struct rte_swx_table_ops *table_ops,
10264                           int *is_stub)
10265 {
10266         struct table *t;
10267
10268         if (!p || (table_id >= p->n_tables))
10269                 return -EINVAL;
10270
10271         t = table_find_by_id(p, table_id);
10272         if (!t)
10273                 return -EINVAL;
10274
10275         if (t->type) {
10276                 if (table_ops)
10277                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10278                 *is_stub = 0;
10279         } else {
10280                 *is_stub = 1;
10281         }
10282
10283         return 0;
10284 }
10285
10286 int
10287 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10288                               uint32_t selector_id,
10289                               struct rte_swx_ctl_selector_info *selector)
10290 {
10291         struct selector *s = NULL;
10292
10293         if (!p || !selector)
10294                 return -EINVAL;
10295
10296         s = selector_find_by_id(p, selector_id);
10297         if (!s)
10298                 return -EINVAL;
10299
10300         strcpy(selector->name, s->name);
10301
10302         selector->n_selector_fields = s->n_selector_fields;
10303         selector->n_groups_max = s->n_groups_max;
10304         selector->n_members_per_group_max = s->n_members_per_group_max;
10305
10306         return 0;
10307 }
10308
10309 int
10310 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10311          uint32_t selector_id,
10312          struct rte_swx_ctl_table_match_field_info *field)
10313 {
10314         struct selector *s;
10315
10316         if (!p || (selector_id >= p->n_selectors) || !field)
10317                 return -EINVAL;
10318
10319         s = selector_find_by_id(p, selector_id);
10320         if (!s)
10321                 return -EINVAL;
10322
10323         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10324         field->is_header = 0;
10325         field->n_bits = s->group_id_field->n_bits;
10326         field->offset = s->group_id_field->offset;
10327
10328         return 0;
10329 }
10330
10331 int
10332 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10333          uint32_t selector_id,
10334          uint32_t selector_field_id,
10335          struct rte_swx_ctl_table_match_field_info *field)
10336 {
10337         struct selector *s;
10338         struct field *f;
10339
10340         if (!p || (selector_id >= p->n_selectors) || !field)
10341                 return -EINVAL;
10342
10343         s = selector_find_by_id(p, selector_id);
10344         if (!s || (selector_field_id >= s->n_selector_fields))
10345                 return -EINVAL;
10346
10347         f = s->selector_fields[selector_field_id];
10348         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10349         field->is_header = s->selector_header ? 1 : 0;
10350         field->n_bits = f->n_bits;
10351         field->offset = f->offset;
10352
10353         return 0;
10354 }
10355
10356 int
10357 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10358          uint32_t selector_id,
10359          struct rte_swx_ctl_table_match_field_info *field)
10360 {
10361         struct selector *s;
10362
10363         if (!p || (selector_id >= p->n_selectors) || !field)
10364                 return -EINVAL;
10365
10366         s = selector_find_by_id(p, selector_id);
10367         if (!s)
10368                 return -EINVAL;
10369
10370         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10371         field->is_header = 0;
10372         field->n_bits = s->member_id_field->n_bits;
10373         field->offset = s->member_id_field->offset;
10374
10375         return 0;
10376 }
10377
10378 int
10379 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10380                              uint32_t learner_id,
10381                              struct rte_swx_ctl_learner_info *learner)
10382 {
10383         struct learner *l = NULL;
10384
10385         if (!p || !learner)
10386                 return -EINVAL;
10387
10388         l = learner_find_by_id(p, learner_id);
10389         if (!l)
10390                 return -EINVAL;
10391
10392         strcpy(learner->name, l->name);
10393
10394         learner->n_match_fields = l->n_fields;
10395         learner->n_actions = l->n_actions;
10396         learner->default_action_is_const = l->default_action_is_const;
10397         learner->size = l->size;
10398
10399         return 0;
10400 }
10401
10402 int
10403 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10404                                          uint32_t learner_id,
10405                                          uint32_t match_field_id,
10406                                          struct rte_swx_ctl_table_match_field_info *match_field)
10407 {
10408         struct learner *l;
10409         struct field *f;
10410
10411         if (!p || (learner_id >= p->n_learners) || !match_field)
10412                 return -EINVAL;
10413
10414         l = learner_find_by_id(p, learner_id);
10415         if (!l || (match_field_id >= l->n_fields))
10416                 return -EINVAL;
10417
10418         f = l->fields[match_field_id];
10419         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10420         match_field->is_header = l->header ? 1 : 0;
10421         match_field->n_bits = f->n_bits;
10422         match_field->offset = f->offset;
10423
10424         return 0;
10425 }
10426
10427 int
10428 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10429                                     uint32_t learner_id,
10430                                     uint32_t learner_action_id,
10431                                     struct rte_swx_ctl_table_action_info *learner_action)
10432 {
10433         struct learner *l;
10434
10435         if (!p || (learner_id >= p->n_learners) || !learner_action)
10436                 return -EINVAL;
10437
10438         l = learner_find_by_id(p, learner_id);
10439         if (!l || (learner_action_id >= l->n_actions))
10440                 return -EINVAL;
10441
10442         learner_action->action_id = l->actions[learner_action_id]->id;
10443
10444         return 0;
10445 }
10446
10447 int
10448 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10449                                  struct rte_swx_table_state **table_state)
10450 {
10451         if (!p || !table_state || !p->build_done)
10452                 return -EINVAL;
10453
10454         *table_state = p->table_state;
10455         return 0;
10456 }
10457
10458 int
10459 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10460                                  struct rte_swx_table_state *table_state)
10461 {
10462         if (!p || !table_state || !p->build_done)
10463                 return -EINVAL;
10464
10465         p->table_state = table_state;
10466         return 0;
10467 }
10468
10469 int
10470 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10471                                         uint32_t port_id,
10472                                         struct rte_swx_port_in_stats *stats)
10473 {
10474         struct port_in *port;
10475
10476         if (!p || !stats)
10477                 return -EINVAL;
10478
10479         port = port_in_find(p, port_id);
10480         if (!port)
10481                 return -EINVAL;
10482
10483         port->type->ops.stats_read(port->obj, stats);
10484         return 0;
10485 }
10486
10487 int
10488 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10489                                          uint32_t port_id,
10490                                          struct rte_swx_port_out_stats *stats)
10491 {
10492         struct port_out *port;
10493
10494         if (!p || !stats)
10495                 return -EINVAL;
10496
10497         port = port_out_find(p, port_id);
10498         if (!port)
10499                 return -EINVAL;
10500
10501         port->type->ops.stats_read(port->obj, stats);
10502         return 0;
10503 }
10504
10505 int
10506 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10507                                       const char *table_name,
10508                                       struct rte_swx_table_stats *stats)
10509 {
10510         struct table *table;
10511         struct table_statistics *table_stats;
10512
10513         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10514                 return -EINVAL;
10515
10516         table = table_find(p, table_name);
10517         if (!table)
10518                 return -EINVAL;
10519
10520         table_stats = &p->table_stats[table->id];
10521
10522         memcpy(stats->n_pkts_action,
10523                table_stats->n_pkts_action,
10524                p->n_actions * sizeof(uint64_t));
10525
10526         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10527         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10528
10529         return 0;
10530 }
10531
10532 int
10533 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10534         const char *selector_name,
10535         struct rte_swx_pipeline_selector_stats *stats)
10536 {
10537         struct selector *s;
10538
10539         if (!p || !selector_name || !selector_name[0] || !stats)
10540                 return -EINVAL;
10541
10542         s = selector_find(p, selector_name);
10543         if (!s)
10544                 return -EINVAL;
10545
10546         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10547
10548         return 0;
10549 }
10550
10551 int
10552 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10553                                         const char *learner_name,
10554                                         struct rte_swx_learner_stats *stats)
10555 {
10556         struct learner *l;
10557         struct learner_statistics *learner_stats;
10558
10559         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10560                 return -EINVAL;
10561
10562         l = learner_find(p, learner_name);
10563         if (!l)
10564                 return -EINVAL;
10565
10566         learner_stats = &p->learner_stats[l->id];
10567
10568         memcpy(stats->n_pkts_action,
10569                learner_stats->n_pkts_action,
10570                p->n_actions * sizeof(uint64_t));
10571
10572         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10573         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10574
10575         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10576         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10577
10578         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10579
10580         return 0;
10581 }
10582
10583 int
10584 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10585                               uint32_t regarray_id,
10586                               struct rte_swx_ctl_regarray_info *regarray)
10587 {
10588         struct regarray *r;
10589
10590         if (!p || !regarray)
10591                 return -EINVAL;
10592
10593         r = regarray_find_by_id(p, regarray_id);
10594         if (!r)
10595                 return -EINVAL;
10596
10597         strcpy(regarray->name, r->name);
10598         regarray->size = r->size;
10599         return 0;
10600 }
10601
10602 int
10603 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10604                                    const char *regarray_name,
10605                                    uint32_t regarray_index,
10606                                    uint64_t *value)
10607 {
10608         struct regarray *regarray;
10609         struct regarray_runtime *r;
10610
10611         if (!p || !regarray_name || !value)
10612                 return -EINVAL;
10613
10614         regarray = regarray_find(p, regarray_name);
10615         if (!regarray || (regarray_index >= regarray->size))
10616                 return -EINVAL;
10617
10618         r = &p->regarray_runtime[regarray->id];
10619         *value = r->regarray[regarray_index];
10620         return 0;
10621 }
10622
10623 int
10624 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10625                                    const char *regarray_name,
10626                                    uint32_t regarray_index,
10627                                    uint64_t value)
10628 {
10629         struct regarray *regarray;
10630         struct regarray_runtime *r;
10631
10632         if (!p || !regarray_name)
10633                 return -EINVAL;
10634
10635         regarray = regarray_find(p, regarray_name);
10636         if (!regarray || (regarray_index >= regarray->size))
10637                 return -EINVAL;
10638
10639         r = &p->regarray_runtime[regarray->id];
10640         r->regarray[regarray_index] = value;
10641         return 0;
10642 }
10643
10644 int
10645 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10646                               uint32_t metarray_id,
10647                               struct rte_swx_ctl_metarray_info *metarray)
10648 {
10649         struct metarray *m;
10650
10651         if (!p || !metarray)
10652                 return -EINVAL;
10653
10654         m = metarray_find_by_id(p, metarray_id);
10655         if (!m)
10656                 return -EINVAL;
10657
10658         strcpy(metarray->name, m->name);
10659         metarray->size = m->size;
10660         return 0;
10661 }
10662
10663 int
10664 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10665                               const char *name,
10666                               struct rte_meter_trtcm_params *params)
10667 {
10668         struct meter_profile *mp;
10669         int status;
10670
10671         CHECK(p, EINVAL);
10672         CHECK_NAME(name, EINVAL);
10673         CHECK(params, EINVAL);
10674         CHECK(!meter_profile_find(p, name), EEXIST);
10675
10676         /* Node allocation. */
10677         mp = calloc(1, sizeof(struct meter_profile));
10678         CHECK(mp, ENOMEM);
10679
10680         /* Node initialization. */
10681         strcpy(mp->name, name);
10682         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10683         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10684         if (status) {
10685                 free(mp);
10686                 CHECK(0, EINVAL);
10687         }
10688
10689         /* Node add to tailq. */
10690         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10691
10692         return 0;
10693 }
10694
10695 int
10696 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10697                                  const char *name)
10698 {
10699         struct meter_profile *mp;
10700
10701         CHECK(p, EINVAL);
10702         CHECK_NAME(name, EINVAL);
10703
10704         mp = meter_profile_find(p, name);
10705         CHECK(mp, EINVAL);
10706         CHECK(!mp->n_users, EBUSY);
10707
10708         /* Remove node from tailq. */
10709         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10710         free(mp);
10711
10712         return 0;
10713 }
10714
10715 int
10716 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10717                         const char *metarray_name,
10718                         uint32_t metarray_index)
10719 {
10720         struct meter_profile *mp_old;
10721         struct metarray *metarray;
10722         struct metarray_runtime *metarray_runtime;
10723         struct meter *m;
10724
10725         CHECK(p, EINVAL);
10726         CHECK_NAME(metarray_name, EINVAL);
10727
10728         metarray = metarray_find(p, metarray_name);
10729         CHECK(metarray, EINVAL);
10730         CHECK(metarray_index < metarray->size, EINVAL);
10731
10732         metarray_runtime = &p->metarray_runtime[metarray->id];
10733         m = &metarray_runtime->metarray[metarray_index];
10734         mp_old = m->profile;
10735
10736         meter_init(m);
10737
10738         mp_old->n_users--;
10739
10740         return 0;
10741 }
10742
10743 int
10744 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10745                       const char *metarray_name,
10746                       uint32_t metarray_index,
10747                       const char *profile_name)
10748 {
10749         struct meter_profile *mp, *mp_old;
10750         struct metarray *metarray;
10751         struct metarray_runtime *metarray_runtime;
10752         struct meter *m;
10753
10754         CHECK(p, EINVAL);
10755         CHECK_NAME(metarray_name, EINVAL);
10756
10757         metarray = metarray_find(p, metarray_name);
10758         CHECK(metarray, EINVAL);
10759         CHECK(metarray_index < metarray->size, EINVAL);
10760
10761         mp = meter_profile_find(p, profile_name);
10762         CHECK(mp, EINVAL);
10763
10764         metarray_runtime = &p->metarray_runtime[metarray->id];
10765         m = &metarray_runtime->metarray[metarray_index];
10766         mp_old = m->profile;
10767
10768         memset(m, 0, sizeof(struct meter));
10769         rte_meter_trtcm_config(&m->m, &mp->profile);
10770         m->profile = mp;
10771         m->color_mask = RTE_COLORS;
10772
10773         mp->n_users++;
10774         mp_old->n_users--;
10775
10776         return 0;
10777 }
10778
10779 int
10780 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10781                              const char *metarray_name,
10782                              uint32_t metarray_index,
10783                              struct rte_swx_ctl_meter_stats *stats)
10784 {
10785         struct metarray *metarray;
10786         struct metarray_runtime *metarray_runtime;
10787         struct meter *m;
10788
10789         CHECK(p, EINVAL);
10790         CHECK_NAME(metarray_name, EINVAL);
10791
10792         metarray = metarray_find(p, metarray_name);
10793         CHECK(metarray, EINVAL);
10794         CHECK(metarray_index < metarray->size, EINVAL);
10795
10796         CHECK(stats, EINVAL);
10797
10798         metarray_runtime = &p->metarray_runtime[metarray->id];
10799         m = &metarray_runtime->metarray[metarray_index];
10800
10801         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10802         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10803
10804         return 0;
10805 }