lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t selector_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107         struct selector_runtime *selector = &t->selectors[selector_id];
2108         struct selector_statistics *stats = &p->selector_stats[selector_id];
2109         uint64_t n_pkts = stats->n_pkts;
2110         int done;
2111
2112         /* Table. */
2113         done = rte_swx_table_selector_select(ts->obj,
2114                            selector->mailbox,
2115                            selector->group_id_buffer,
2116                            selector->selector_buffer,
2117                            selector->member_id_buffer);
2118         if (!done) {
2119                 /* Thread. */
2120                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2121                       p->thread_id,
2122                       selector_id);
2123
2124                 thread_yield(p);
2125                 return;
2126         }
2127
2128
2129         TRACE("[Thread %2u] selector %u\n",
2130               p->thread_id,
2131               selector_id);
2132
2133         stats->n_pkts = n_pkts + 1;
2134
2135         /* Thread. */
2136         thread_ip_inc(p);
2137 }
2138
2139 static inline void
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t learner_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146                 p->n_selectors + learner_id];
2147         struct learner_runtime *l = &t->learners[learner_id];
2148         struct learner_statistics *stats = &p->learner_stats[learner_id];
2149         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150         uint8_t *action_data;
2151         int done, hit;
2152
2153         /* Table. */
2154         time = rte_get_tsc_cycles();
2155
2156         done = rte_swx_table_learner_lookup(ts->obj,
2157                                             l->mailbox,
2158                                             time,
2159                                             l->key,
2160                                             &action_id,
2161                                             &action_data,
2162                                             &hit);
2163         if (!done) {
2164                 /* Thread. */
2165                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2166                       p->thread_id,
2167                       learner_id);
2168
2169                 thread_yield(p);
2170                 return;
2171         }
2172
2173         action_id = hit ? action_id : ts->default_action_id;
2174         action_data = hit ? action_data : ts->default_action_data;
2175         n_pkts_hit = stats->n_pkts_hit[hit];
2176         n_pkts_action = stats->n_pkts_action[action_id];
2177
2178         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2179               p->thread_id,
2180               learner_id,
2181               hit ? "hit" : "miss",
2182               (uint32_t)action_id);
2183
2184         t->action_id = action_id;
2185         t->structs[0] = action_data;
2186         t->hit = hit;
2187         t->learner_id = learner_id;
2188         t->time = time;
2189         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2191
2192         /* Thread. */
2193         thread_ip_action_call(p, t, action_id);
2194 }
2195
2196 /*
2197  * learn.
2198  */
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2201
2202 static int
2203 action_has_nbo_args(struct action *a);
2204
2205 static int
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207                       struct action *action,
2208                       char **tokens,
2209                       int n_tokens,
2210                       struct instruction *instr,
2211                       struct instruction_data *data __rte_unused)
2212 {
2213         struct action *a;
2214
2215         CHECK(action, EINVAL);
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         a = action_find(p, tokens[1]);
2219         CHECK(a, EINVAL);
2220         CHECK(!action_has_nbo_args(a), EINVAL);
2221
2222         instr->type = INSTR_LEARNER_LEARN;
2223         instr->learn.action_id = a->id;
2224
2225         return 0;
2226 }
2227
2228 static inline void
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_learn_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_inc(p);
2238 }
2239
2240 /*
2241  * forget.
2242  */
2243 static int
2244 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2245                        struct action *action,
2246                        char **tokens __rte_unused,
2247                        int n_tokens,
2248                        struct instruction *instr,
2249                        struct instruction_data *data __rte_unused)
2250 {
2251         CHECK(action, EINVAL);
2252         CHECK(n_tokens == 1, EINVAL);
2253
2254         instr->type = INSTR_LEARNER_FORGET;
2255
2256         return 0;
2257 }
2258
2259 static inline void
2260 instr_forget_exec(struct rte_swx_pipeline *p)
2261 {
2262         struct thread *t = &p->threads[p->thread_id];
2263         struct instruction *ip = t->ip;
2264
2265         __instr_forget_exec(p, t, ip);
2266
2267         /* Thread. */
2268         thread_ip_inc(p);
2269 }
2270
2271 /*
2272  * extern.
2273  */
2274 static int
2275 instr_extern_translate(struct rte_swx_pipeline *p,
2276                        struct action *action __rte_unused,
2277                        char **tokens,
2278                        int n_tokens,
2279                        struct instruction *instr,
2280                        struct instruction_data *data __rte_unused)
2281 {
2282         char *token = tokens[1];
2283
2284         CHECK(n_tokens == 2, EINVAL);
2285
2286         if (token[0] == 'e') {
2287                 struct extern_obj *obj;
2288                 struct extern_type_member_func *func;
2289
2290                 func = extern_obj_member_func_parse(p, token, &obj);
2291                 CHECK(func, EINVAL);
2292
2293                 instr->type = INSTR_EXTERN_OBJ;
2294                 instr->ext_obj.ext_obj_id = obj->id;
2295                 instr->ext_obj.func_id = func->id;
2296
2297                 return 0;
2298         }
2299
2300         if (token[0] == 'f') {
2301                 struct extern_func *func;
2302
2303                 func = extern_func_parse(p, token);
2304                 CHECK(func, EINVAL);
2305
2306                 instr->type = INSTR_EXTERN_FUNC;
2307                 instr->ext_func.ext_func_id = func->id;
2308
2309                 return 0;
2310         }
2311
2312         CHECK(0, EINVAL);
2313 }
2314
2315 static inline void
2316 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2317 {
2318         struct thread *t = &p->threads[p->thread_id];
2319         struct instruction *ip = t->ip;
2320         uint32_t done;
2321
2322         /* Extern object member function execute. */
2323         done = __instr_extern_obj_exec(p, t, ip);
2324
2325         /* Thread. */
2326         thread_ip_inc_cond(t, done);
2327         thread_yield_cond(p, done ^ 1);
2328 }
2329
2330 static inline void
2331 instr_extern_func_exec(struct rte_swx_pipeline *p)
2332 {
2333         struct thread *t = &p->threads[p->thread_id];
2334         struct instruction *ip = t->ip;
2335         uint32_t done;
2336
2337         /* Extern function execute. */
2338         done = __instr_extern_func_exec(p, t, ip);
2339
2340         /* Thread. */
2341         thread_ip_inc_cond(t, done);
2342         thread_yield_cond(p, done ^ 1);
2343 }
2344
2345 /*
2346  * mov.
2347  */
2348 static int
2349 instr_mov_translate(struct rte_swx_pipeline *p,
2350                     struct action *action,
2351                     char **tokens,
2352                     int n_tokens,
2353                     struct instruction *instr,
2354                     struct instruction_data *data __rte_unused)
2355 {
2356         char *dst = tokens[1], *src = tokens[2];
2357         struct field *fdst, *fsrc;
2358         uint64_t src_val;
2359         uint32_t dst_struct_id = 0, src_struct_id = 0;
2360
2361         CHECK(n_tokens == 3, EINVAL);
2362
2363         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2364         CHECK(fdst, EINVAL);
2365         CHECK(!fdst->var_size, EINVAL);
2366
2367         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2368         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2369         if (fsrc) {
2370                 CHECK(!fsrc->var_size, EINVAL);
2371
2372                 instr->type = INSTR_MOV;
2373                 if (dst[0] != 'h' && src[0] == 'h')
2374                         instr->type = INSTR_MOV_MH;
2375                 if (dst[0] == 'h' && src[0] != 'h')
2376                         instr->type = INSTR_MOV_HM;
2377                 if (dst[0] == 'h' && src[0] == 'h')
2378                         instr->type = INSTR_MOV_HH;
2379
2380                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2381                 instr->mov.dst.n_bits = fdst->n_bits;
2382                 instr->mov.dst.offset = fdst->offset / 8;
2383                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2384                 instr->mov.src.n_bits = fsrc->n_bits;
2385                 instr->mov.src.offset = fsrc->offset / 8;
2386                 return 0;
2387         }
2388
2389         /* MOV_I. */
2390         src_val = strtoull(src, &src, 0);
2391         CHECK(!src[0], EINVAL);
2392
2393         if (dst[0] == 'h')
2394                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2395
2396         instr->type = INSTR_MOV_I;
2397         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2398         instr->mov.dst.n_bits = fdst->n_bits;
2399         instr->mov.dst.offset = fdst->offset / 8;
2400         instr->mov.src_val = src_val;
2401         return 0;
2402 }
2403
2404 static inline void
2405 instr_mov_exec(struct rte_swx_pipeline *p)
2406 {
2407         struct thread *t = &p->threads[p->thread_id];
2408         struct instruction *ip = t->ip;
2409
2410         __instr_mov_exec(p, t, ip);
2411
2412         /* Thread. */
2413         thread_ip_inc(p);
2414 }
2415
2416 static inline void
2417 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2418 {
2419         struct thread *t = &p->threads[p->thread_id];
2420         struct instruction *ip = t->ip;
2421
2422         __instr_mov_mh_exec(p, t, ip);
2423
2424         /* Thread. */
2425         thread_ip_inc(p);
2426 }
2427
2428 static inline void
2429 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2430 {
2431         struct thread *t = &p->threads[p->thread_id];
2432         struct instruction *ip = t->ip;
2433
2434         __instr_mov_hm_exec(p, t, ip);
2435
2436         /* Thread. */
2437         thread_ip_inc(p);
2438 }
2439
2440 static inline void
2441 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2442 {
2443         struct thread *t = &p->threads[p->thread_id];
2444         struct instruction *ip = t->ip;
2445
2446         __instr_mov_hh_exec(p, t, ip);
2447
2448         /* Thread. */
2449         thread_ip_inc(p);
2450 }
2451
2452 static inline void
2453 instr_mov_i_exec(struct rte_swx_pipeline *p)
2454 {
2455         struct thread *t = &p->threads[p->thread_id];
2456         struct instruction *ip = t->ip;
2457
2458         __instr_mov_i_exec(p, t, ip);
2459
2460         /* Thread. */
2461         thread_ip_inc(p);
2462 }
2463
2464 /*
2465  * dma.
2466  */
2467 static inline void
2468 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2469 {
2470         struct thread *t = &p->threads[p->thread_id];
2471         struct instruction *ip = t->ip;
2472
2473         __instr_dma_ht_exec(p, t, ip);
2474
2475         /* Thread. */
2476         thread_ip_inc(p);
2477 }
2478
2479 static inline void
2480 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2481 {
2482         struct thread *t = &p->threads[p->thread_id];
2483         struct instruction *ip = t->ip;
2484
2485         __instr_dma_ht2_exec(p, t, ip);
2486
2487         /* Thread. */
2488         thread_ip_inc(p);
2489 }
2490
2491 static inline void
2492 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2493 {
2494         struct thread *t = &p->threads[p->thread_id];
2495         struct instruction *ip = t->ip;
2496
2497         __instr_dma_ht3_exec(p, t, ip);
2498
2499         /* Thread. */
2500         thread_ip_inc(p);
2501 }
2502
2503 static inline void
2504 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2505 {
2506         struct thread *t = &p->threads[p->thread_id];
2507         struct instruction *ip = t->ip;
2508
2509         __instr_dma_ht4_exec(p, t, ip);
2510
2511         /* Thread. */
2512         thread_ip_inc(p);
2513 }
2514
2515 static inline void
2516 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2517 {
2518         struct thread *t = &p->threads[p->thread_id];
2519         struct instruction *ip = t->ip;
2520
2521         __instr_dma_ht5_exec(p, t, ip);
2522
2523         /* Thread. */
2524         thread_ip_inc(p);
2525 }
2526
2527 static inline void
2528 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2529 {
2530         struct thread *t = &p->threads[p->thread_id];
2531         struct instruction *ip = t->ip;
2532
2533         __instr_dma_ht6_exec(p, t, ip);
2534
2535         /* Thread. */
2536         thread_ip_inc(p);
2537 }
2538
2539 static inline void
2540 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2541 {
2542         struct thread *t = &p->threads[p->thread_id];
2543         struct instruction *ip = t->ip;
2544
2545         __instr_dma_ht7_exec(p, t, ip);
2546
2547         /* Thread. */
2548         thread_ip_inc(p);
2549 }
2550
2551 static inline void
2552 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2553 {
2554         struct thread *t = &p->threads[p->thread_id];
2555         struct instruction *ip = t->ip;
2556
2557         __instr_dma_ht8_exec(p, t, ip);
2558
2559         /* Thread. */
2560         thread_ip_inc(p);
2561 }
2562
2563 /*
2564  * alu.
2565  */
2566 static int
2567 instr_alu_add_translate(struct rte_swx_pipeline *p,
2568                         struct action *action,
2569                         char **tokens,
2570                         int n_tokens,
2571                         struct instruction *instr,
2572                         struct instruction_data *data __rte_unused)
2573 {
2574         char *dst = tokens[1], *src = tokens[2];
2575         struct field *fdst, *fsrc;
2576         uint64_t src_val;
2577         uint32_t dst_struct_id = 0, src_struct_id = 0;
2578
2579         CHECK(n_tokens == 3, EINVAL);
2580
2581         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2582         CHECK(fdst, EINVAL);
2583         CHECK(!fdst->var_size, EINVAL);
2584
2585         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2586         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2587         if (fsrc) {
2588                 CHECK(!fsrc->var_size, EINVAL);
2589
2590                 instr->type = INSTR_ALU_ADD;
2591                 if (dst[0] == 'h' && src[0] != 'h')
2592                         instr->type = INSTR_ALU_ADD_HM;
2593                 if (dst[0] != 'h' && src[0] == 'h')
2594                         instr->type = INSTR_ALU_ADD_MH;
2595                 if (dst[0] == 'h' && src[0] == 'h')
2596                         instr->type = INSTR_ALU_ADD_HH;
2597
2598                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2599                 instr->alu.dst.n_bits = fdst->n_bits;
2600                 instr->alu.dst.offset = fdst->offset / 8;
2601                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2602                 instr->alu.src.n_bits = fsrc->n_bits;
2603                 instr->alu.src.offset = fsrc->offset / 8;
2604                 return 0;
2605         }
2606
2607         /* ADD_MI, ADD_HI. */
2608         src_val = strtoull(src, &src, 0);
2609         CHECK(!src[0], EINVAL);
2610
2611         instr->type = INSTR_ALU_ADD_MI;
2612         if (dst[0] == 'h')
2613                 instr->type = INSTR_ALU_ADD_HI;
2614
2615         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2616         instr->alu.dst.n_bits = fdst->n_bits;
2617         instr->alu.dst.offset = fdst->offset / 8;
2618         instr->alu.src_val = src_val;
2619         return 0;
2620 }
2621
2622 static int
2623 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2624                         struct action *action,
2625                         char **tokens,
2626                         int n_tokens,
2627                         struct instruction *instr,
2628                         struct instruction_data *data __rte_unused)
2629 {
2630         char *dst = tokens[1], *src = tokens[2];
2631         struct field *fdst, *fsrc;
2632         uint64_t src_val;
2633         uint32_t dst_struct_id = 0, src_struct_id = 0;
2634
2635         CHECK(n_tokens == 3, EINVAL);
2636
2637         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2638         CHECK(fdst, EINVAL);
2639         CHECK(!fdst->var_size, EINVAL);
2640
2641         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2642         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2643         if (fsrc) {
2644                 CHECK(!fsrc->var_size, EINVAL);
2645
2646                 instr->type = INSTR_ALU_SUB;
2647                 if (dst[0] == 'h' && src[0] != 'h')
2648                         instr->type = INSTR_ALU_SUB_HM;
2649                 if (dst[0] != 'h' && src[0] == 'h')
2650                         instr->type = INSTR_ALU_SUB_MH;
2651                 if (dst[0] == 'h' && src[0] == 'h')
2652                         instr->type = INSTR_ALU_SUB_HH;
2653
2654                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2655                 instr->alu.dst.n_bits = fdst->n_bits;
2656                 instr->alu.dst.offset = fdst->offset / 8;
2657                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2658                 instr->alu.src.n_bits = fsrc->n_bits;
2659                 instr->alu.src.offset = fsrc->offset / 8;
2660                 return 0;
2661         }
2662
2663         /* SUB_MI, SUB_HI. */
2664         src_val = strtoull(src, &src, 0);
2665         CHECK(!src[0], EINVAL);
2666
2667         instr->type = INSTR_ALU_SUB_MI;
2668         if (dst[0] == 'h')
2669                 instr->type = INSTR_ALU_SUB_HI;
2670
2671         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2672         instr->alu.dst.n_bits = fdst->n_bits;
2673         instr->alu.dst.offset = fdst->offset / 8;
2674         instr->alu.src_val = src_val;
2675         return 0;
2676 }
2677
2678 static int
2679 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2680                           struct action *action __rte_unused,
2681                           char **tokens,
2682                           int n_tokens,
2683                           struct instruction *instr,
2684                           struct instruction_data *data __rte_unused)
2685 {
2686         char *dst = tokens[1], *src = tokens[2];
2687         struct header *hdst, *hsrc;
2688         struct field *fdst, *fsrc;
2689
2690         CHECK(n_tokens == 3, EINVAL);
2691
2692         fdst = header_field_parse(p, dst, &hdst);
2693         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2694         CHECK(!fdst->var_size, EINVAL);
2695
2696         /* CKADD_FIELD. */
2697         fsrc = header_field_parse(p, src, &hsrc);
2698         if (fsrc) {
2699                 CHECK(!fsrc->var_size, EINVAL);
2700
2701                 instr->type = INSTR_ALU_CKADD_FIELD;
2702                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2703                 instr->alu.dst.n_bits = fdst->n_bits;
2704                 instr->alu.dst.offset = fdst->offset / 8;
2705                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2706                 instr->alu.src.n_bits = fsrc->n_bits;
2707                 instr->alu.src.offset = fsrc->offset / 8;
2708                 return 0;
2709         }
2710
2711         /* CKADD_STRUCT, CKADD_STRUCT20. */
2712         hsrc = header_parse(p, src);
2713         CHECK(hsrc, EINVAL);
2714         CHECK(!hsrc->st->var_size, EINVAL);
2715
2716         instr->type = INSTR_ALU_CKADD_STRUCT;
2717         if ((hsrc->st->n_bits / 8) == 20)
2718                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2719
2720         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2721         instr->alu.dst.n_bits = fdst->n_bits;
2722         instr->alu.dst.offset = fdst->offset / 8;
2723         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2724         instr->alu.src.n_bits = hsrc->st->n_bits;
2725         instr->alu.src.offset = 0; /* Unused. */
2726         return 0;
2727 }
2728
2729 static int
2730 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2731                           struct action *action __rte_unused,
2732                           char **tokens,
2733                           int n_tokens,
2734                           struct instruction *instr,
2735                           struct instruction_data *data __rte_unused)
2736 {
2737         char *dst = tokens[1], *src = tokens[2];
2738         struct header *hdst, *hsrc;
2739         struct field *fdst, *fsrc;
2740
2741         CHECK(n_tokens == 3, EINVAL);
2742
2743         fdst = header_field_parse(p, dst, &hdst);
2744         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2745         CHECK(!fdst->var_size, EINVAL);
2746
2747         fsrc = header_field_parse(p, src, &hsrc);
2748         CHECK(fsrc, EINVAL);
2749         CHECK(!fsrc->var_size, EINVAL);
2750
2751         instr->type = INSTR_ALU_CKSUB_FIELD;
2752         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2753         instr->alu.dst.n_bits = fdst->n_bits;
2754         instr->alu.dst.offset = fdst->offset / 8;
2755         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2756         instr->alu.src.n_bits = fsrc->n_bits;
2757         instr->alu.src.offset = fsrc->offset / 8;
2758         return 0;
2759 }
2760
2761 static int
2762 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2763                         struct action *action,
2764                         char **tokens,
2765                         int n_tokens,
2766                         struct instruction *instr,
2767                         struct instruction_data *data __rte_unused)
2768 {
2769         char *dst = tokens[1], *src = tokens[2];
2770         struct field *fdst, *fsrc;
2771         uint64_t src_val;
2772         uint32_t dst_struct_id = 0, src_struct_id = 0;
2773
2774         CHECK(n_tokens == 3, EINVAL);
2775
2776         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2777         CHECK(fdst, EINVAL);
2778         CHECK(!fdst->var_size, EINVAL);
2779
2780         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2781         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2782         if (fsrc) {
2783                 CHECK(!fsrc->var_size, EINVAL);
2784
2785                 instr->type = INSTR_ALU_SHL;
2786                 if (dst[0] == 'h' && src[0] != 'h')
2787                         instr->type = INSTR_ALU_SHL_HM;
2788                 if (dst[0] != 'h' && src[0] == 'h')
2789                         instr->type = INSTR_ALU_SHL_MH;
2790                 if (dst[0] == 'h' && src[0] == 'h')
2791                         instr->type = INSTR_ALU_SHL_HH;
2792
2793                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2794                 instr->alu.dst.n_bits = fdst->n_bits;
2795                 instr->alu.dst.offset = fdst->offset / 8;
2796                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2797                 instr->alu.src.n_bits = fsrc->n_bits;
2798                 instr->alu.src.offset = fsrc->offset / 8;
2799                 return 0;
2800         }
2801
2802         /* SHL_MI, SHL_HI. */
2803         src_val = strtoull(src, &src, 0);
2804         CHECK(!src[0], EINVAL);
2805
2806         instr->type = INSTR_ALU_SHL_MI;
2807         if (dst[0] == 'h')
2808                 instr->type = INSTR_ALU_SHL_HI;
2809
2810         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2811         instr->alu.dst.n_bits = fdst->n_bits;
2812         instr->alu.dst.offset = fdst->offset / 8;
2813         instr->alu.src_val = src_val;
2814         return 0;
2815 }
2816
2817 static int
2818 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2819                         struct action *action,
2820                         char **tokens,
2821                         int n_tokens,
2822                         struct instruction *instr,
2823                         struct instruction_data *data __rte_unused)
2824 {
2825         char *dst = tokens[1], *src = tokens[2];
2826         struct field *fdst, *fsrc;
2827         uint64_t src_val;
2828         uint32_t dst_struct_id = 0, src_struct_id = 0;
2829
2830         CHECK(n_tokens == 3, EINVAL);
2831
2832         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2833         CHECK(fdst, EINVAL);
2834         CHECK(!fdst->var_size, EINVAL);
2835
2836         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2837         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2838         if (fsrc) {
2839                 CHECK(!fsrc->var_size, EINVAL);
2840
2841                 instr->type = INSTR_ALU_SHR;
2842                 if (dst[0] == 'h' && src[0] != 'h')
2843                         instr->type = INSTR_ALU_SHR_HM;
2844                 if (dst[0] != 'h' && src[0] == 'h')
2845                         instr->type = INSTR_ALU_SHR_MH;
2846                 if (dst[0] == 'h' && src[0] == 'h')
2847                         instr->type = INSTR_ALU_SHR_HH;
2848
2849                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2850                 instr->alu.dst.n_bits = fdst->n_bits;
2851                 instr->alu.dst.offset = fdst->offset / 8;
2852                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2853                 instr->alu.src.n_bits = fsrc->n_bits;
2854                 instr->alu.src.offset = fsrc->offset / 8;
2855                 return 0;
2856         }
2857
2858         /* SHR_MI, SHR_HI. */
2859         src_val = strtoull(src, &src, 0);
2860         CHECK(!src[0], EINVAL);
2861
2862         instr->type = INSTR_ALU_SHR_MI;
2863         if (dst[0] == 'h')
2864                 instr->type = INSTR_ALU_SHR_HI;
2865
2866         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2867         instr->alu.dst.n_bits = fdst->n_bits;
2868         instr->alu.dst.offset = fdst->offset / 8;
2869         instr->alu.src_val = src_val;
2870         return 0;
2871 }
2872
2873 static int
2874 instr_alu_and_translate(struct rte_swx_pipeline *p,
2875                         struct action *action,
2876                         char **tokens,
2877                         int n_tokens,
2878                         struct instruction *instr,
2879                         struct instruction_data *data __rte_unused)
2880 {
2881         char *dst = tokens[1], *src = tokens[2];
2882         struct field *fdst, *fsrc;
2883         uint64_t src_val;
2884         uint32_t dst_struct_id = 0, src_struct_id = 0;
2885
2886         CHECK(n_tokens == 3, EINVAL);
2887
2888         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2889         CHECK(fdst, EINVAL);
2890         CHECK(!fdst->var_size, EINVAL);
2891
2892         /* AND, AND_MH, AND_HM, AND_HH. */
2893         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2894         if (fsrc) {
2895                 CHECK(!fsrc->var_size, EINVAL);
2896
2897                 instr->type = INSTR_ALU_AND;
2898                 if (dst[0] != 'h' && src[0] == 'h')
2899                         instr->type = INSTR_ALU_AND_MH;
2900                 if (dst[0] == 'h' && src[0] != 'h')
2901                         instr->type = INSTR_ALU_AND_HM;
2902                 if (dst[0] == 'h' && src[0] == 'h')
2903                         instr->type = INSTR_ALU_AND_HH;
2904
2905                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2906                 instr->alu.dst.n_bits = fdst->n_bits;
2907                 instr->alu.dst.offset = fdst->offset / 8;
2908                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2909                 instr->alu.src.n_bits = fsrc->n_bits;
2910                 instr->alu.src.offset = fsrc->offset / 8;
2911                 return 0;
2912         }
2913
2914         /* AND_I. */
2915         src_val = strtoull(src, &src, 0);
2916         CHECK(!src[0], EINVAL);
2917
2918         if (dst[0] == 'h')
2919                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2920
2921         instr->type = INSTR_ALU_AND_I;
2922         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2923         instr->alu.dst.n_bits = fdst->n_bits;
2924         instr->alu.dst.offset = fdst->offset / 8;
2925         instr->alu.src_val = src_val;
2926         return 0;
2927 }
2928
2929 static int
2930 instr_alu_or_translate(struct rte_swx_pipeline *p,
2931                        struct action *action,
2932                        char **tokens,
2933                        int n_tokens,
2934                        struct instruction *instr,
2935                        struct instruction_data *data __rte_unused)
2936 {
2937         char *dst = tokens[1], *src = tokens[2];
2938         struct field *fdst, *fsrc;
2939         uint64_t src_val;
2940         uint32_t dst_struct_id = 0, src_struct_id = 0;
2941
2942         CHECK(n_tokens == 3, EINVAL);
2943
2944         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2945         CHECK(fdst, EINVAL);
2946         CHECK(!fdst->var_size, EINVAL);
2947
2948         /* OR, OR_MH, OR_HM, OR_HH. */
2949         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2950         if (fsrc) {
2951                 CHECK(!fsrc->var_size, EINVAL);
2952
2953                 instr->type = INSTR_ALU_OR;
2954                 if (dst[0] != 'h' && src[0] == 'h')
2955                         instr->type = INSTR_ALU_OR_MH;
2956                 if (dst[0] == 'h' && src[0] != 'h')
2957                         instr->type = INSTR_ALU_OR_HM;
2958                 if (dst[0] == 'h' && src[0] == 'h')
2959                         instr->type = INSTR_ALU_OR_HH;
2960
2961                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2962                 instr->alu.dst.n_bits = fdst->n_bits;
2963                 instr->alu.dst.offset = fdst->offset / 8;
2964                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2965                 instr->alu.src.n_bits = fsrc->n_bits;
2966                 instr->alu.src.offset = fsrc->offset / 8;
2967                 return 0;
2968         }
2969
2970         /* OR_I. */
2971         src_val = strtoull(src, &src, 0);
2972         CHECK(!src[0], EINVAL);
2973
2974         if (dst[0] == 'h')
2975                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2976
2977         instr->type = INSTR_ALU_OR_I;
2978         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2979         instr->alu.dst.n_bits = fdst->n_bits;
2980         instr->alu.dst.offset = fdst->offset / 8;
2981         instr->alu.src_val = src_val;
2982         return 0;
2983 }
2984
2985 static int
2986 instr_alu_xor_translate(struct rte_swx_pipeline *p,
2987                         struct action *action,
2988                         char **tokens,
2989                         int n_tokens,
2990                         struct instruction *instr,
2991                         struct instruction_data *data __rte_unused)
2992 {
2993         char *dst = tokens[1], *src = tokens[2];
2994         struct field *fdst, *fsrc;
2995         uint64_t src_val;
2996         uint32_t dst_struct_id = 0, src_struct_id = 0;
2997
2998         CHECK(n_tokens == 3, EINVAL);
2999
3000         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3001         CHECK(fdst, EINVAL);
3002         CHECK(!fdst->var_size, EINVAL);
3003
3004         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3006         if (fsrc) {
3007                 CHECK(!fsrc->var_size, EINVAL);
3008
3009                 instr->type = INSTR_ALU_XOR;
3010                 if (dst[0] != 'h' && src[0] == 'h')
3011                         instr->type = INSTR_ALU_XOR_MH;
3012                 if (dst[0] == 'h' && src[0] != 'h')
3013                         instr->type = INSTR_ALU_XOR_HM;
3014                 if (dst[0] == 'h' && src[0] == 'h')
3015                         instr->type = INSTR_ALU_XOR_HH;
3016
3017                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3018                 instr->alu.dst.n_bits = fdst->n_bits;
3019                 instr->alu.dst.offset = fdst->offset / 8;
3020                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3021                 instr->alu.src.n_bits = fsrc->n_bits;
3022                 instr->alu.src.offset = fsrc->offset / 8;
3023                 return 0;
3024         }
3025
3026         /* XOR_I. */
3027         src_val = strtoull(src, &src, 0);
3028         CHECK(!src[0], EINVAL);
3029
3030         if (dst[0] == 'h')
3031                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3032
3033         instr->type = INSTR_ALU_XOR_I;
3034         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3035         instr->alu.dst.n_bits = fdst->n_bits;
3036         instr->alu.dst.offset = fdst->offset / 8;
3037         instr->alu.src_val = src_val;
3038         return 0;
3039 }
3040
3041 static inline void
3042 instr_alu_add_exec(struct rte_swx_pipeline *p)
3043 {
3044         struct thread *t = &p->threads[p->thread_id];
3045         struct instruction *ip = t->ip;
3046
3047         /* Structs */
3048         __instr_alu_add_exec(p, t, ip);
3049
3050         /* Thread. */
3051         thread_ip_inc(p);
3052 }
3053
3054 static inline void
3055 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3056 {
3057         struct thread *t = &p->threads[p->thread_id];
3058         struct instruction *ip = t->ip;
3059
3060         /* Structs. */
3061         __instr_alu_add_mh_exec(p, t, ip);
3062
3063         /* Thread. */
3064         thread_ip_inc(p);
3065 }
3066
3067 static inline void
3068 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3069 {
3070         struct thread *t = &p->threads[p->thread_id];
3071         struct instruction *ip = t->ip;
3072
3073         /* Structs. */
3074         __instr_alu_add_hm_exec(p, t, ip);
3075
3076         /* Thread. */
3077         thread_ip_inc(p);
3078 }
3079
3080 static inline void
3081 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3082 {
3083         struct thread *t = &p->threads[p->thread_id];
3084         struct instruction *ip = t->ip;
3085
3086         /* Structs. */
3087         __instr_alu_add_hh_exec(p, t, ip);
3088
3089         /* Thread. */
3090         thread_ip_inc(p);
3091 }
3092
3093 static inline void
3094 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3095 {
3096         struct thread *t = &p->threads[p->thread_id];
3097         struct instruction *ip = t->ip;
3098
3099         /* Structs. */
3100         __instr_alu_add_mi_exec(p, t, ip);
3101
3102         /* Thread. */
3103         thread_ip_inc(p);
3104 }
3105
3106 static inline void
3107 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3108 {
3109         struct thread *t = &p->threads[p->thread_id];
3110         struct instruction *ip = t->ip;
3111
3112         /* Structs. */
3113         __instr_alu_add_hi_exec(p, t, ip);
3114
3115         /* Thread. */
3116         thread_ip_inc(p);
3117 }
3118
3119 static inline void
3120 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3121 {
3122         struct thread *t = &p->threads[p->thread_id];
3123         struct instruction *ip = t->ip;
3124
3125         /* Structs. */
3126         __instr_alu_sub_exec(p, t, ip);
3127
3128         /* Thread. */
3129         thread_ip_inc(p);
3130 }
3131
3132 static inline void
3133 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3134 {
3135         struct thread *t = &p->threads[p->thread_id];
3136         struct instruction *ip = t->ip;
3137
3138         /* Structs. */
3139         __instr_alu_sub_mh_exec(p, t, ip);
3140
3141         /* Thread. */
3142         thread_ip_inc(p);
3143 }
3144
3145 static inline void
3146 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3147 {
3148         struct thread *t = &p->threads[p->thread_id];
3149         struct instruction *ip = t->ip;
3150
3151         /* Structs. */
3152         __instr_alu_sub_hm_exec(p, t, ip);
3153
3154         /* Thread. */
3155         thread_ip_inc(p);
3156 }
3157
3158 static inline void
3159 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3160 {
3161         struct thread *t = &p->threads[p->thread_id];
3162         struct instruction *ip = t->ip;
3163
3164         /* Structs. */
3165         __instr_alu_sub_hh_exec(p, t, ip);
3166
3167         /* Thread. */
3168         thread_ip_inc(p);
3169 }
3170
3171 static inline void
3172 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3173 {
3174         struct thread *t = &p->threads[p->thread_id];
3175         struct instruction *ip = t->ip;
3176
3177         /* Structs. */
3178         __instr_alu_sub_mi_exec(p, t, ip);
3179
3180         /* Thread. */
3181         thread_ip_inc(p);
3182 }
3183
3184 static inline void
3185 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3186 {
3187         struct thread *t = &p->threads[p->thread_id];
3188         struct instruction *ip = t->ip;
3189
3190         /* Structs. */
3191         __instr_alu_sub_hi_exec(p, t, ip);
3192
3193         /* Thread. */
3194         thread_ip_inc(p);
3195 }
3196
3197 static inline void
3198 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3199 {
3200         struct thread *t = &p->threads[p->thread_id];
3201         struct instruction *ip = t->ip;
3202
3203         /* Structs. */
3204         __instr_alu_shl_exec(p, t, ip);
3205
3206         /* Thread. */
3207         thread_ip_inc(p);
3208 }
3209
3210 static inline void
3211 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3212 {
3213         struct thread *t = &p->threads[p->thread_id];
3214         struct instruction *ip = t->ip;
3215
3216         /* Structs. */
3217         __instr_alu_shl_mh_exec(p, t, ip);
3218
3219         /* Thread. */
3220         thread_ip_inc(p);
3221 }
3222
3223 static inline void
3224 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3225 {
3226         struct thread *t = &p->threads[p->thread_id];
3227         struct instruction *ip = t->ip;
3228
3229         /* Structs. */
3230         __instr_alu_shl_hm_exec(p, t, ip);
3231
3232         /* Thread. */
3233         thread_ip_inc(p);
3234 }
3235
3236 static inline void
3237 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3238 {
3239         struct thread *t = &p->threads[p->thread_id];
3240         struct instruction *ip = t->ip;
3241
3242         /* Structs. */
3243         __instr_alu_shl_hh_exec(p, t, ip);
3244
3245         /* Thread. */
3246         thread_ip_inc(p);
3247 }
3248
3249 static inline void
3250 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3251 {
3252         struct thread *t = &p->threads[p->thread_id];
3253         struct instruction *ip = t->ip;
3254
3255         /* Structs. */
3256         __instr_alu_shl_mi_exec(p, t, ip);
3257
3258         /* Thread. */
3259         thread_ip_inc(p);
3260 }
3261
3262 static inline void
3263 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3264 {
3265         struct thread *t = &p->threads[p->thread_id];
3266         struct instruction *ip = t->ip;
3267
3268         /* Structs. */
3269         __instr_alu_shl_hi_exec(p, t, ip);
3270
3271         /* Thread. */
3272         thread_ip_inc(p);
3273 }
3274
3275 static inline void
3276 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3277 {
3278         struct thread *t = &p->threads[p->thread_id];
3279         struct instruction *ip = t->ip;
3280
3281         /* Structs. */
3282         __instr_alu_shr_exec(p, t, ip);
3283
3284         /* Thread. */
3285         thread_ip_inc(p);
3286 }
3287
3288 static inline void
3289 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3290 {
3291         struct thread *t = &p->threads[p->thread_id];
3292         struct instruction *ip = t->ip;
3293
3294         /* Structs. */
3295         __instr_alu_shr_mh_exec(p, t, ip);
3296
3297         /* Thread. */
3298         thread_ip_inc(p);
3299 }
3300
3301 static inline void
3302 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3303 {
3304         struct thread *t = &p->threads[p->thread_id];
3305         struct instruction *ip = t->ip;
3306
3307         /* Structs. */
3308         __instr_alu_shr_hm_exec(p, t, ip);
3309
3310         /* Thread. */
3311         thread_ip_inc(p);
3312 }
3313
3314 static inline void
3315 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3316 {
3317         struct thread *t = &p->threads[p->thread_id];
3318         struct instruction *ip = t->ip;
3319
3320         /* Structs. */
3321         __instr_alu_shr_hh_exec(p, t, ip);
3322
3323         /* Thread. */
3324         thread_ip_inc(p);
3325 }
3326
3327 static inline void
3328 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3329 {
3330         struct thread *t = &p->threads[p->thread_id];
3331         struct instruction *ip = t->ip;
3332
3333         /* Structs. */
3334         __instr_alu_shr_mi_exec(p, t, ip);
3335
3336         /* Thread. */
3337         thread_ip_inc(p);
3338 }
3339
3340 static inline void
3341 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3342 {
3343         struct thread *t = &p->threads[p->thread_id];
3344         struct instruction *ip = t->ip;
3345
3346         /* Structs. */
3347         __instr_alu_shr_hi_exec(p, t, ip);
3348
3349         /* Thread. */
3350         thread_ip_inc(p);
3351 }
3352
3353 static inline void
3354 instr_alu_and_exec(struct rte_swx_pipeline *p)
3355 {
3356         struct thread *t = &p->threads[p->thread_id];
3357         struct instruction *ip = t->ip;
3358
3359         /* Structs. */
3360         __instr_alu_and_exec(p, t, ip);
3361
3362         /* Thread. */
3363         thread_ip_inc(p);
3364 }
3365
3366 static inline void
3367 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3368 {
3369         struct thread *t = &p->threads[p->thread_id];
3370         struct instruction *ip = t->ip;
3371
3372         /* Structs. */
3373         __instr_alu_and_mh_exec(p, t, ip);
3374
3375         /* Thread. */
3376         thread_ip_inc(p);
3377 }
3378
3379 static inline void
3380 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3381 {
3382         struct thread *t = &p->threads[p->thread_id];
3383         struct instruction *ip = t->ip;
3384
3385         /* Structs. */
3386         __instr_alu_and_hm_exec(p, t, ip);
3387
3388         /* Thread. */
3389         thread_ip_inc(p);
3390 }
3391
3392 static inline void
3393 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3394 {
3395         struct thread *t = &p->threads[p->thread_id];
3396         struct instruction *ip = t->ip;
3397
3398         /* Structs. */
3399         __instr_alu_and_hh_exec(p, t, ip);
3400
3401         /* Thread. */
3402         thread_ip_inc(p);
3403 }
3404
3405 static inline void
3406 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3407 {
3408         struct thread *t = &p->threads[p->thread_id];
3409         struct instruction *ip = t->ip;
3410
3411         /* Structs. */
3412         __instr_alu_and_i_exec(p, t, ip);
3413
3414         /* Thread. */
3415         thread_ip_inc(p);
3416 }
3417
3418 static inline void
3419 instr_alu_or_exec(struct rte_swx_pipeline *p)
3420 {
3421         struct thread *t = &p->threads[p->thread_id];
3422         struct instruction *ip = t->ip;
3423
3424         /* Structs. */
3425         __instr_alu_or_exec(p, t, ip);
3426
3427         /* Thread. */
3428         thread_ip_inc(p);
3429 }
3430
3431 static inline void
3432 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3433 {
3434         struct thread *t = &p->threads[p->thread_id];
3435         struct instruction *ip = t->ip;
3436
3437         /* Structs. */
3438         __instr_alu_or_mh_exec(p, t, ip);
3439
3440         /* Thread. */
3441         thread_ip_inc(p);
3442 }
3443
3444 static inline void
3445 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3446 {
3447         struct thread *t = &p->threads[p->thread_id];
3448         struct instruction *ip = t->ip;
3449
3450         /* Structs. */
3451         __instr_alu_or_hm_exec(p, t, ip);
3452
3453         /* Thread. */
3454         thread_ip_inc(p);
3455 }
3456
3457 static inline void
3458 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3459 {
3460         struct thread *t = &p->threads[p->thread_id];
3461         struct instruction *ip = t->ip;
3462
3463         /* Structs. */
3464         __instr_alu_or_hh_exec(p, t, ip);
3465
3466         /* Thread. */
3467         thread_ip_inc(p);
3468 }
3469
3470 static inline void
3471 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3472 {
3473         struct thread *t = &p->threads[p->thread_id];
3474         struct instruction *ip = t->ip;
3475
3476         /* Structs. */
3477         __instr_alu_or_i_exec(p, t, ip);
3478
3479         /* Thread. */
3480         thread_ip_inc(p);
3481 }
3482
3483 static inline void
3484 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3485 {
3486         struct thread *t = &p->threads[p->thread_id];
3487         struct instruction *ip = t->ip;
3488
3489         /* Structs. */
3490         __instr_alu_xor_exec(p, t, ip);
3491
3492         /* Thread. */
3493         thread_ip_inc(p);
3494 }
3495
3496 static inline void
3497 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3498 {
3499         struct thread *t = &p->threads[p->thread_id];
3500         struct instruction *ip = t->ip;
3501
3502         /* Structs. */
3503         __instr_alu_xor_mh_exec(p, t, ip);
3504
3505         /* Thread. */
3506         thread_ip_inc(p);
3507 }
3508
3509 static inline void
3510 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3511 {
3512         struct thread *t = &p->threads[p->thread_id];
3513         struct instruction *ip = t->ip;
3514
3515         /* Structs. */
3516         __instr_alu_xor_hm_exec(p, t, ip);
3517
3518         /* Thread. */
3519         thread_ip_inc(p);
3520 }
3521
3522 static inline void
3523 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3524 {
3525         struct thread *t = &p->threads[p->thread_id];
3526         struct instruction *ip = t->ip;
3527
3528         /* Structs. */
3529         __instr_alu_xor_hh_exec(p, t, ip);
3530
3531         /* Thread. */
3532         thread_ip_inc(p);
3533 }
3534
3535 static inline void
3536 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3537 {
3538         struct thread *t = &p->threads[p->thread_id];
3539         struct instruction *ip = t->ip;
3540
3541         /* Structs. */
3542         __instr_alu_xor_i_exec(p, t, ip);
3543
3544         /* Thread. */
3545         thread_ip_inc(p);
3546 }
3547
3548 static inline void
3549 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3550 {
3551         struct thread *t = &p->threads[p->thread_id];
3552         struct instruction *ip = t->ip;
3553
3554         /* Structs. */
3555         __instr_alu_ckadd_field_exec(p, t, ip);
3556
3557         /* Thread. */
3558         thread_ip_inc(p);
3559 }
3560
3561 static inline void
3562 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3563 {
3564         struct thread *t = &p->threads[p->thread_id];
3565         struct instruction *ip = t->ip;
3566
3567         /* Structs. */
3568         __instr_alu_cksub_field_exec(p, t, ip);
3569
3570         /* Thread. */
3571         thread_ip_inc(p);
3572 }
3573
3574 static inline void
3575 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3576 {
3577         struct thread *t = &p->threads[p->thread_id];
3578         struct instruction *ip = t->ip;
3579
3580         /* Structs. */
3581         __instr_alu_ckadd_struct20_exec(p, t, ip);
3582
3583         /* Thread. */
3584         thread_ip_inc(p);
3585 }
3586
3587 static inline void
3588 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3589 {
3590         struct thread *t = &p->threads[p->thread_id];
3591         struct instruction *ip = t->ip;
3592
3593         /* Structs. */
3594         __instr_alu_ckadd_struct_exec(p, t, ip);
3595
3596         /* Thread. */
3597         thread_ip_inc(p);
3598 }
3599
3600 /*
3601  * Register array.
3602  */
3603 static struct regarray *
3604 regarray_find(struct rte_swx_pipeline *p, const char *name);
3605
3606 static int
3607 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3608                       struct action *action,
3609                       char **tokens,
3610                       int n_tokens,
3611                       struct instruction *instr,
3612                       struct instruction_data *data __rte_unused)
3613 {
3614         char *regarray = tokens[1], *idx = tokens[2];
3615         struct regarray *r;
3616         struct field *fidx;
3617         uint32_t idx_struct_id, idx_val;
3618
3619         CHECK(n_tokens == 3, EINVAL);
3620
3621         r = regarray_find(p, regarray);
3622         CHECK(r, EINVAL);
3623
3624         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3625         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3626         if (fidx) {
3627                 CHECK(!fidx->var_size, EINVAL);
3628
3629                 instr->type = INSTR_REGPREFETCH_RM;
3630                 if (idx[0] == 'h')
3631                         instr->type = INSTR_REGPREFETCH_RH;
3632
3633                 instr->regarray.regarray_id = r->id;
3634                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3635                 instr->regarray.idx.n_bits = fidx->n_bits;
3636                 instr->regarray.idx.offset = fidx->offset / 8;
3637                 instr->regarray.dstsrc_val = 0; /* Unused. */
3638                 return 0;
3639         }
3640
3641         /* REGPREFETCH_RI. */
3642         idx_val = strtoul(idx, &idx, 0);
3643         CHECK(!idx[0], EINVAL);
3644
3645         instr->type = INSTR_REGPREFETCH_RI;
3646         instr->regarray.regarray_id = r->id;
3647         instr->regarray.idx_val = idx_val;
3648         instr->regarray.dstsrc_val = 0; /* Unused. */
3649         return 0;
3650 }
3651
3652 static int
3653 instr_regrd_translate(struct rte_swx_pipeline *p,
3654                       struct action *action,
3655                       char **tokens,
3656                       int n_tokens,
3657                       struct instruction *instr,
3658                       struct instruction_data *data __rte_unused)
3659 {
3660         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3661         struct regarray *r;
3662         struct field *fdst, *fidx;
3663         uint32_t dst_struct_id, idx_struct_id, idx_val;
3664
3665         CHECK(n_tokens == 4, EINVAL);
3666
3667         r = regarray_find(p, regarray);
3668         CHECK(r, EINVAL);
3669
3670         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3671         CHECK(fdst, EINVAL);
3672         CHECK(!fdst->var_size, EINVAL);
3673
3674         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3675         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3676         if (fidx) {
3677                 CHECK(!fidx->var_size, EINVAL);
3678
3679                 instr->type = INSTR_REGRD_MRM;
3680                 if (dst[0] == 'h' && idx[0] != 'h')
3681                         instr->type = INSTR_REGRD_HRM;
3682                 if (dst[0] != 'h' && idx[0] == 'h')
3683                         instr->type = INSTR_REGRD_MRH;
3684                 if (dst[0] == 'h' && idx[0] == 'h')
3685                         instr->type = INSTR_REGRD_HRH;
3686
3687                 instr->regarray.regarray_id = r->id;
3688                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3689                 instr->regarray.idx.n_bits = fidx->n_bits;
3690                 instr->regarray.idx.offset = fidx->offset / 8;
3691                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3692                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3693                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3694                 return 0;
3695         }
3696
3697         /* REGRD_MRI, REGRD_HRI. */
3698         idx_val = strtoul(idx, &idx, 0);
3699         CHECK(!idx[0], EINVAL);
3700
3701         instr->type = INSTR_REGRD_MRI;
3702         if (dst[0] == 'h')
3703                 instr->type = INSTR_REGRD_HRI;
3704
3705         instr->regarray.regarray_id = r->id;
3706         instr->regarray.idx_val = idx_val;
3707         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3708         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3709         instr->regarray.dstsrc.offset = fdst->offset / 8;
3710         return 0;
3711 }
3712
3713 static int
3714 instr_regwr_translate(struct rte_swx_pipeline *p,
3715                       struct action *action,
3716                       char **tokens,
3717                       int n_tokens,
3718                       struct instruction *instr,
3719                       struct instruction_data *data __rte_unused)
3720 {
3721         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3722         struct regarray *r;
3723         struct field *fidx, *fsrc;
3724         uint64_t src_val;
3725         uint32_t idx_struct_id, idx_val, src_struct_id;
3726
3727         CHECK(n_tokens == 4, EINVAL);
3728
3729         r = regarray_find(p, regarray);
3730         CHECK(r, EINVAL);
3731
3732         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3733         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3734         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3735         if (fidx && fsrc) {
3736                 CHECK(!fidx->var_size, EINVAL);
3737                 CHECK(!fsrc->var_size, EINVAL);
3738
3739                 instr->type = INSTR_REGWR_RMM;
3740                 if (idx[0] == 'h' && src[0] != 'h')
3741                         instr->type = INSTR_REGWR_RHM;
3742                 if (idx[0] != 'h' && src[0] == 'h')
3743                         instr->type = INSTR_REGWR_RMH;
3744                 if (idx[0] == 'h' && src[0] == 'h')
3745                         instr->type = INSTR_REGWR_RHH;
3746
3747                 instr->regarray.regarray_id = r->id;
3748                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3749                 instr->regarray.idx.n_bits = fidx->n_bits;
3750                 instr->regarray.idx.offset = fidx->offset / 8;
3751                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3752                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3753                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3754                 return 0;
3755         }
3756
3757         /* REGWR_RHI, REGWR_RMI. */
3758         if (fidx && !fsrc) {
3759                 CHECK(!fidx->var_size, EINVAL);
3760
3761                 src_val = strtoull(src, &src, 0);
3762                 CHECK(!src[0], EINVAL);
3763
3764                 instr->type = INSTR_REGWR_RMI;
3765                 if (idx[0] == 'h')
3766                         instr->type = INSTR_REGWR_RHI;
3767
3768                 instr->regarray.regarray_id = r->id;
3769                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3770                 instr->regarray.idx.n_bits = fidx->n_bits;
3771                 instr->regarray.idx.offset = fidx->offset / 8;
3772                 instr->regarray.dstsrc_val = src_val;
3773                 return 0;
3774         }
3775
3776         /* REGWR_RIH, REGWR_RIM. */
3777         if (!fidx && fsrc) {
3778                 idx_val = strtoul(idx, &idx, 0);
3779                 CHECK(!idx[0], EINVAL);
3780
3781                 CHECK(!fsrc->var_size, EINVAL);
3782
3783                 instr->type = INSTR_REGWR_RIM;
3784                 if (src[0] == 'h')
3785                         instr->type = INSTR_REGWR_RIH;
3786
3787                 instr->regarray.regarray_id = r->id;
3788                 instr->regarray.idx_val = idx_val;
3789                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3790                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3791                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3792                 return 0;
3793         }
3794
3795         /* REGWR_RII. */
3796         src_val = strtoull(src, &src, 0);
3797         CHECK(!src[0], EINVAL);
3798
3799         idx_val = strtoul(idx, &idx, 0);
3800         CHECK(!idx[0], EINVAL);
3801
3802         instr->type = INSTR_REGWR_RII;
3803         instr->regarray.idx_val = idx_val;
3804         instr->regarray.dstsrc_val = src_val;
3805
3806         return 0;
3807 }
3808
3809 static int
3810 instr_regadd_translate(struct rte_swx_pipeline *p,
3811                        struct action *action,
3812                        char **tokens,
3813                        int n_tokens,
3814                        struct instruction *instr,
3815                        struct instruction_data *data __rte_unused)
3816 {
3817         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3818         struct regarray *r;
3819         struct field *fidx, *fsrc;
3820         uint64_t src_val;
3821         uint32_t idx_struct_id, idx_val, src_struct_id;
3822
3823         CHECK(n_tokens == 4, EINVAL);
3824
3825         r = regarray_find(p, regarray);
3826         CHECK(r, EINVAL);
3827
3828         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3829         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3830         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3831         if (fidx && fsrc) {
3832                 CHECK(!fidx->var_size, EINVAL);
3833                 CHECK(!fsrc->var_size, EINVAL);
3834
3835                 instr->type = INSTR_REGADD_RMM;
3836                 if (idx[0] == 'h' && src[0] != 'h')
3837                         instr->type = INSTR_REGADD_RHM;
3838                 if (idx[0] != 'h' && src[0] == 'h')
3839                         instr->type = INSTR_REGADD_RMH;
3840                 if (idx[0] == 'h' && src[0] == 'h')
3841                         instr->type = INSTR_REGADD_RHH;
3842
3843                 instr->regarray.regarray_id = r->id;
3844                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3845                 instr->regarray.idx.n_bits = fidx->n_bits;
3846                 instr->regarray.idx.offset = fidx->offset / 8;
3847                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3848                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3849                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3850                 return 0;
3851         }
3852
3853         /* REGADD_RHI, REGADD_RMI. */
3854         if (fidx && !fsrc) {
3855                 CHECK(!fidx->var_size, EINVAL);
3856
3857                 src_val = strtoull(src, &src, 0);
3858                 CHECK(!src[0], EINVAL);
3859
3860                 instr->type = INSTR_REGADD_RMI;
3861                 if (idx[0] == 'h')
3862                         instr->type = INSTR_REGADD_RHI;
3863
3864                 instr->regarray.regarray_id = r->id;
3865                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3866                 instr->regarray.idx.n_bits = fidx->n_bits;
3867                 instr->regarray.idx.offset = fidx->offset / 8;
3868                 instr->regarray.dstsrc_val = src_val;
3869                 return 0;
3870         }
3871
3872         /* REGADD_RIH, REGADD_RIM. */
3873         if (!fidx && fsrc) {
3874                 idx_val = strtoul(idx, &idx, 0);
3875                 CHECK(!idx[0], EINVAL);
3876
3877                 CHECK(!fsrc->var_size, EINVAL);
3878
3879                 instr->type = INSTR_REGADD_RIM;
3880                 if (src[0] == 'h')
3881                         instr->type = INSTR_REGADD_RIH;
3882
3883                 instr->regarray.regarray_id = r->id;
3884                 instr->regarray.idx_val = idx_val;
3885                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3886                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3887                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3888                 return 0;
3889         }
3890
3891         /* REGADD_RII. */
3892         src_val = strtoull(src, &src, 0);
3893         CHECK(!src[0], EINVAL);
3894
3895         idx_val = strtoul(idx, &idx, 0);
3896         CHECK(!idx[0], EINVAL);
3897
3898         instr->type = INSTR_REGADD_RII;
3899         instr->regarray.idx_val = idx_val;
3900         instr->regarray.dstsrc_val = src_val;
3901         return 0;
3902 }
3903
3904 static inline uint64_t *
3905 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
3906 {
3907         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3908         return r->regarray;
3909 }
3910
3911 static inline uint64_t
3912 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
3913 {
3914         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3915
3916         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3917         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3918         uint64_t idx64 = *idx64_ptr;
3919         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3920         uint64_t idx = idx64 & idx64_mask & r->size_mask;
3921
3922         return idx;
3923 }
3924
3925 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3926
3927 static inline uint64_t
3928 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
3929 {
3930         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3931
3932         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3933         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3934         uint64_t idx64 = *idx64_ptr;
3935         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3936
3937         return idx;
3938 }
3939
3940 #else
3941
3942 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
3943
3944 #endif
3945
3946 static inline uint64_t
3947 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
3948 {
3949         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3950
3951         uint64_t idx = ip->regarray.idx_val & r->size_mask;
3952
3953         return idx;
3954 }
3955
3956 static inline uint64_t
3957 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
3958 {
3959         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3960         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3961         uint64_t src64 = *src64_ptr;
3962         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3963         uint64_t src = src64 & src64_mask;
3964
3965         return src;
3966 }
3967
3968 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3969
3970 static inline uint64_t
3971 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
3972 {
3973         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3974         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3975         uint64_t src64 = *src64_ptr;
3976         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3977
3978         return src;
3979 }
3980
3981 #else
3982
3983 #define instr_regarray_src_nbo instr_regarray_src_hbo
3984
3985 #endif
3986
3987 static inline void
3988 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
3989 {
3990         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3991         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3992         uint64_t dst64 = *dst64_ptr;
3993         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3994
3995         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3996
3997 }
3998
3999 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4000
4001 static inline void
4002 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4003 {
4004         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4005         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4006         uint64_t dst64 = *dst64_ptr;
4007         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4008
4009         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4010         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4011 }
4012
4013 #else
4014
4015 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4016
4017 #endif
4018
4019 static inline void
4020 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4021 {
4022         struct thread *t = &p->threads[p->thread_id];
4023         struct instruction *ip = t->ip;
4024         uint64_t *regarray, idx;
4025
4026         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4027
4028         /* Structs. */
4029         regarray = instr_regarray_regarray(p, ip);
4030         idx = instr_regarray_idx_nbo(p, t, ip);
4031         rte_prefetch0(&regarray[idx]);
4032
4033         /* Thread. */
4034         thread_ip_inc(p);
4035 }
4036
4037 static inline void
4038 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4039 {
4040         struct thread *t = &p->threads[p->thread_id];
4041         struct instruction *ip = t->ip;
4042         uint64_t *regarray, idx;
4043
4044         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4045
4046         /* Structs. */
4047         regarray = instr_regarray_regarray(p, ip);
4048         idx = instr_regarray_idx_hbo(p, t, ip);
4049         rte_prefetch0(&regarray[idx]);
4050
4051         /* Thread. */
4052         thread_ip_inc(p);
4053 }
4054
4055 static inline void
4056 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4057 {
4058         struct thread *t = &p->threads[p->thread_id];
4059         struct instruction *ip = t->ip;
4060         uint64_t *regarray, idx;
4061
4062         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4063
4064         /* Structs. */
4065         regarray = instr_regarray_regarray(p, ip);
4066         idx = instr_regarray_idx_imm(p, ip);
4067         rte_prefetch0(&regarray[idx]);
4068
4069         /* Thread. */
4070         thread_ip_inc(p);
4071 }
4072
4073 static inline void
4074 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4075 {
4076         struct thread *t = &p->threads[p->thread_id];
4077         struct instruction *ip = t->ip;
4078         uint64_t *regarray, idx;
4079
4080         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4081
4082         /* Structs. */
4083         regarray = instr_regarray_regarray(p, ip);
4084         idx = instr_regarray_idx_nbo(p, t, ip);
4085         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4086
4087         /* Thread. */
4088         thread_ip_inc(p);
4089 }
4090
4091 static inline void
4092 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4093 {
4094         struct thread *t = &p->threads[p->thread_id];
4095         struct instruction *ip = t->ip;
4096         uint64_t *regarray, idx;
4097
4098         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4099
4100         /* Structs. */
4101         regarray = instr_regarray_regarray(p, ip);
4102         idx = instr_regarray_idx_hbo(p, t, ip);
4103         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4104
4105         /* Thread. */
4106         thread_ip_inc(p);
4107 }
4108
4109 static inline void
4110 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4111 {
4112         struct thread *t = &p->threads[p->thread_id];
4113         struct instruction *ip = t->ip;
4114         uint64_t *regarray, idx;
4115
4116         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4117
4118         /* Structs. */
4119         regarray = instr_regarray_regarray(p, ip);
4120         idx = instr_regarray_idx_nbo(p, t, ip);
4121         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4122
4123         /* Thread. */
4124         thread_ip_inc(p);
4125 }
4126
4127 static inline void
4128 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4129 {
4130         struct thread *t = &p->threads[p->thread_id];
4131         struct instruction *ip = t->ip;
4132         uint64_t *regarray, idx;
4133
4134         /* Structs. */
4135         regarray = instr_regarray_regarray(p, ip);
4136         idx = instr_regarray_idx_hbo(p, t, ip);
4137         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4138
4139         /* Thread. */
4140         thread_ip_inc(p);
4141 }
4142
4143 static inline void
4144 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4145 {
4146         struct thread *t = &p->threads[p->thread_id];
4147         struct instruction *ip = t->ip;
4148         uint64_t *regarray, idx;
4149
4150         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4151
4152         /* Structs. */
4153         regarray = instr_regarray_regarray(p, ip);
4154         idx = instr_regarray_idx_imm(p, ip);
4155         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4156
4157         /* Thread. */
4158         thread_ip_inc(p);
4159 }
4160
4161 static inline void
4162 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4163 {
4164         struct thread *t = &p->threads[p->thread_id];
4165         struct instruction *ip = t->ip;
4166         uint64_t *regarray, idx;
4167
4168         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4169
4170         /* Structs. */
4171         regarray = instr_regarray_regarray(p, ip);
4172         idx = instr_regarray_idx_imm(p, ip);
4173         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4174
4175         /* Thread. */
4176         thread_ip_inc(p);
4177 }
4178
4179 static inline void
4180 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4181 {
4182         struct thread *t = &p->threads[p->thread_id];
4183         struct instruction *ip = t->ip;
4184         uint64_t *regarray, idx, src;
4185
4186         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4187
4188         /* Structs. */
4189         regarray = instr_regarray_regarray(p, ip);
4190         idx = instr_regarray_idx_nbo(p, t, ip);
4191         src = instr_regarray_src_nbo(t, ip);
4192         regarray[idx] = src;
4193
4194         /* Thread. */
4195         thread_ip_inc(p);
4196 }
4197
4198 static inline void
4199 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4200 {
4201         struct thread *t = &p->threads[p->thread_id];
4202         struct instruction *ip = t->ip;
4203         uint64_t *regarray, idx, src;
4204
4205         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4206
4207         /* Structs. */
4208         regarray = instr_regarray_regarray(p, ip);
4209         idx = instr_regarray_idx_nbo(p, t, ip);
4210         src = instr_regarray_src_hbo(t, ip);
4211         regarray[idx] = src;
4212
4213         /* Thread. */
4214         thread_ip_inc(p);
4215 }
4216
4217 static inline void
4218 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4219 {
4220         struct thread *t = &p->threads[p->thread_id];
4221         struct instruction *ip = t->ip;
4222         uint64_t *regarray, idx, src;
4223
4224         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4225
4226         /* Structs. */
4227         regarray = instr_regarray_regarray(p, ip);
4228         idx = instr_regarray_idx_hbo(p, t, ip);
4229         src = instr_regarray_src_nbo(t, ip);
4230         regarray[idx] = src;
4231
4232         /* Thread. */
4233         thread_ip_inc(p);
4234 }
4235
4236 static inline void
4237 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4238 {
4239         struct thread *t = &p->threads[p->thread_id];
4240         struct instruction *ip = t->ip;
4241         uint64_t *regarray, idx, src;
4242
4243         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4244
4245         /* Structs. */
4246         regarray = instr_regarray_regarray(p, ip);
4247         idx = instr_regarray_idx_hbo(p, t, ip);
4248         src = instr_regarray_src_hbo(t, ip);
4249         regarray[idx] = src;
4250
4251         /* Thread. */
4252         thread_ip_inc(p);
4253 }
4254
4255 static inline void
4256 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4257 {
4258         struct thread *t = &p->threads[p->thread_id];
4259         struct instruction *ip = t->ip;
4260         uint64_t *regarray, idx, src;
4261
4262         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4263
4264         /* Structs. */
4265         regarray = instr_regarray_regarray(p, ip);
4266         idx = instr_regarray_idx_nbo(p, t, ip);
4267         src = ip->regarray.dstsrc_val;
4268         regarray[idx] = src;
4269
4270         /* Thread. */
4271         thread_ip_inc(p);
4272 }
4273
4274 static inline void
4275 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4276 {
4277         struct thread *t = &p->threads[p->thread_id];
4278         struct instruction *ip = t->ip;
4279         uint64_t *regarray, idx, src;
4280
4281         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4282
4283         /* Structs. */
4284         regarray = instr_regarray_regarray(p, ip);
4285         idx = instr_regarray_idx_hbo(p, t, ip);
4286         src = ip->regarray.dstsrc_val;
4287         regarray[idx] = src;
4288
4289         /* Thread. */
4290         thread_ip_inc(p);
4291 }
4292
4293 static inline void
4294 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4295 {
4296         struct thread *t = &p->threads[p->thread_id];
4297         struct instruction *ip = t->ip;
4298         uint64_t *regarray, idx, src;
4299
4300         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4301
4302         /* Structs. */
4303         regarray = instr_regarray_regarray(p, ip);
4304         idx = instr_regarray_idx_imm(p, ip);
4305         src = instr_regarray_src_nbo(t, ip);
4306         regarray[idx] = src;
4307
4308         /* Thread. */
4309         thread_ip_inc(p);
4310 }
4311
4312 static inline void
4313 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4314 {
4315         struct thread *t = &p->threads[p->thread_id];
4316         struct instruction *ip = t->ip;
4317         uint64_t *regarray, idx, src;
4318
4319         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4320
4321         /* Structs. */
4322         regarray = instr_regarray_regarray(p, ip);
4323         idx = instr_regarray_idx_imm(p, ip);
4324         src = instr_regarray_src_hbo(t, ip);
4325         regarray[idx] = src;
4326
4327         /* Thread. */
4328         thread_ip_inc(p);
4329 }
4330
4331 static inline void
4332 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4333 {
4334         struct thread *t = &p->threads[p->thread_id];
4335         struct instruction *ip = t->ip;
4336         uint64_t *regarray, idx, src;
4337
4338         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4339
4340         /* Structs. */
4341         regarray = instr_regarray_regarray(p, ip);
4342         idx = instr_regarray_idx_imm(p, ip);
4343         src = ip->regarray.dstsrc_val;
4344         regarray[idx] = src;
4345
4346         /* Thread. */
4347         thread_ip_inc(p);
4348 }
4349
4350 static inline void
4351 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4352 {
4353         struct thread *t = &p->threads[p->thread_id];
4354         struct instruction *ip = t->ip;
4355         uint64_t *regarray, idx, src;
4356
4357         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4358
4359         /* Structs. */
4360         regarray = instr_regarray_regarray(p, ip);
4361         idx = instr_regarray_idx_nbo(p, t, ip);
4362         src = instr_regarray_src_nbo(t, ip);
4363         regarray[idx] += src;
4364
4365         /* Thread. */
4366         thread_ip_inc(p);
4367 }
4368
4369 static inline void
4370 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4371 {
4372         struct thread *t = &p->threads[p->thread_id];
4373         struct instruction *ip = t->ip;
4374         uint64_t *regarray, idx, src;
4375
4376         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4377
4378         /* Structs. */
4379         regarray = instr_regarray_regarray(p, ip);
4380         idx = instr_regarray_idx_nbo(p, t, ip);
4381         src = instr_regarray_src_hbo(t, ip);
4382         regarray[idx] += src;
4383
4384         /* Thread. */
4385         thread_ip_inc(p);
4386 }
4387
4388 static inline void
4389 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4390 {
4391         struct thread *t = &p->threads[p->thread_id];
4392         struct instruction *ip = t->ip;
4393         uint64_t *regarray, idx, src;
4394
4395         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4396
4397         /* Structs. */
4398         regarray = instr_regarray_regarray(p, ip);
4399         idx = instr_regarray_idx_hbo(p, t, ip);
4400         src = instr_regarray_src_nbo(t, ip);
4401         regarray[idx] += src;
4402
4403         /* Thread. */
4404         thread_ip_inc(p);
4405 }
4406
4407 static inline void
4408 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4409 {
4410         struct thread *t = &p->threads[p->thread_id];
4411         struct instruction *ip = t->ip;
4412         uint64_t *regarray, idx, src;
4413
4414         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4415
4416         /* Structs. */
4417         regarray = instr_regarray_regarray(p, ip);
4418         idx = instr_regarray_idx_hbo(p, t, ip);
4419         src = instr_regarray_src_hbo(t, ip);
4420         regarray[idx] += src;
4421
4422         /* Thread. */
4423         thread_ip_inc(p);
4424 }
4425
4426 static inline void
4427 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4428 {
4429         struct thread *t = &p->threads[p->thread_id];
4430         struct instruction *ip = t->ip;
4431         uint64_t *regarray, idx, src;
4432
4433         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4434
4435         /* Structs. */
4436         regarray = instr_regarray_regarray(p, ip);
4437         idx = instr_regarray_idx_nbo(p, t, ip);
4438         src = ip->regarray.dstsrc_val;
4439         regarray[idx] += src;
4440
4441         /* Thread. */
4442         thread_ip_inc(p);
4443 }
4444
4445 static inline void
4446 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4447 {
4448         struct thread *t = &p->threads[p->thread_id];
4449         struct instruction *ip = t->ip;
4450         uint64_t *regarray, idx, src;
4451
4452         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4453
4454         /* Structs. */
4455         regarray = instr_regarray_regarray(p, ip);
4456         idx = instr_regarray_idx_hbo(p, t, ip);
4457         src = ip->regarray.dstsrc_val;
4458         regarray[idx] += src;
4459
4460         /* Thread. */
4461         thread_ip_inc(p);
4462 }
4463
4464 static inline void
4465 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4466 {
4467         struct thread *t = &p->threads[p->thread_id];
4468         struct instruction *ip = t->ip;
4469         uint64_t *regarray, idx, src;
4470
4471         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4472
4473         /* Structs. */
4474         regarray = instr_regarray_regarray(p, ip);
4475         idx = instr_regarray_idx_imm(p, ip);
4476         src = instr_regarray_src_nbo(t, ip);
4477         regarray[idx] += src;
4478
4479         /* Thread. */
4480         thread_ip_inc(p);
4481 }
4482
4483 static inline void
4484 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4485 {
4486         struct thread *t = &p->threads[p->thread_id];
4487         struct instruction *ip = t->ip;
4488         uint64_t *regarray, idx, src;
4489
4490         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4491
4492         /* Structs. */
4493         regarray = instr_regarray_regarray(p, ip);
4494         idx = instr_regarray_idx_imm(p, ip);
4495         src = instr_regarray_src_hbo(t, ip);
4496         regarray[idx] += src;
4497
4498         /* Thread. */
4499         thread_ip_inc(p);
4500 }
4501
4502 static inline void
4503 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4504 {
4505         struct thread *t = &p->threads[p->thread_id];
4506         struct instruction *ip = t->ip;
4507         uint64_t *regarray, idx, src;
4508
4509         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4510
4511         /* Structs. */
4512         regarray = instr_regarray_regarray(p, ip);
4513         idx = instr_regarray_idx_imm(p, ip);
4514         src = ip->regarray.dstsrc_val;
4515         regarray[idx] += src;
4516
4517         /* Thread. */
4518         thread_ip_inc(p);
4519 }
4520
4521 /*
4522  * metarray.
4523  */
4524 static struct metarray *
4525 metarray_find(struct rte_swx_pipeline *p, const char *name);
4526
4527 static int
4528 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4529                             struct action *action,
4530                             char **tokens,
4531                             int n_tokens,
4532                             struct instruction *instr,
4533                             struct instruction_data *data __rte_unused)
4534 {
4535         char *metarray = tokens[1], *idx = tokens[2];
4536         struct metarray *m;
4537         struct field *fidx;
4538         uint32_t idx_struct_id, idx_val;
4539
4540         CHECK(n_tokens == 3, EINVAL);
4541
4542         m = metarray_find(p, metarray);
4543         CHECK(m, EINVAL);
4544
4545         /* METPREFETCH_H, METPREFETCH_M. */
4546         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4547         if (fidx) {
4548                 CHECK(!fidx->var_size, EINVAL);
4549
4550                 instr->type = INSTR_METPREFETCH_M;
4551                 if (idx[0] == 'h')
4552                         instr->type = INSTR_METPREFETCH_H;
4553
4554                 instr->meter.metarray_id = m->id;
4555                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4556                 instr->meter.idx.n_bits = fidx->n_bits;
4557                 instr->meter.idx.offset = fidx->offset / 8;
4558                 return 0;
4559         }
4560
4561         /* METPREFETCH_I. */
4562         idx_val = strtoul(idx, &idx, 0);
4563         CHECK(!idx[0], EINVAL);
4564
4565         instr->type = INSTR_METPREFETCH_I;
4566         instr->meter.metarray_id = m->id;
4567         instr->meter.idx_val = idx_val;
4568         return 0;
4569 }
4570
4571 static int
4572 instr_meter_translate(struct rte_swx_pipeline *p,
4573                       struct action *action,
4574                       char **tokens,
4575                       int n_tokens,
4576                       struct instruction *instr,
4577                       struct instruction_data *data __rte_unused)
4578 {
4579         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4580         char *color_in = tokens[4], *color_out = tokens[5];
4581         struct metarray *m;
4582         struct field *fidx, *flength, *fcin, *fcout;
4583         uint32_t idx_struct_id, length_struct_id;
4584         uint32_t color_in_struct_id, color_out_struct_id;
4585
4586         CHECK(n_tokens == 6, EINVAL);
4587
4588         m = metarray_find(p, metarray);
4589         CHECK(m, EINVAL);
4590
4591         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4592
4593         flength = struct_field_parse(p, action, length, &length_struct_id);
4594         CHECK(flength, EINVAL);
4595         CHECK(!flength->var_size, EINVAL);
4596
4597         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4598
4599         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4600         CHECK(fcout, EINVAL);
4601         CHECK(!fcout->var_size, EINVAL);
4602
4603         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4604         if (fidx && fcin) {
4605                 CHECK(!fidx->var_size, EINVAL);
4606                 CHECK(!fcin->var_size, EINVAL);
4607
4608                 instr->type = INSTR_METER_MMM;
4609                 if (idx[0] == 'h' && length[0] == 'h')
4610                         instr->type = INSTR_METER_HHM;
4611                 if (idx[0] == 'h' && length[0] != 'h')
4612                         instr->type = INSTR_METER_HMM;
4613                 if (idx[0] != 'h' && length[0] == 'h')
4614                         instr->type = INSTR_METER_MHM;
4615
4616                 instr->meter.metarray_id = m->id;
4617
4618                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4619                 instr->meter.idx.n_bits = fidx->n_bits;
4620                 instr->meter.idx.offset = fidx->offset / 8;
4621
4622                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4623                 instr->meter.length.n_bits = flength->n_bits;
4624                 instr->meter.length.offset = flength->offset / 8;
4625
4626                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4627                 instr->meter.color_in.n_bits = fcin->n_bits;
4628                 instr->meter.color_in.offset = fcin->offset / 8;
4629
4630                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4631                 instr->meter.color_out.n_bits = fcout->n_bits;
4632                 instr->meter.color_out.offset = fcout->offset / 8;
4633
4634                 return 0;
4635         }
4636
4637         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4638         if (fidx && !fcin) {
4639                 uint32_t color_in_val;
4640
4641                 CHECK(!fidx->var_size, EINVAL);
4642
4643                 color_in_val = strtoul(color_in, &color_in, 0);
4644                 CHECK(!color_in[0], EINVAL);
4645
4646                 instr->type = INSTR_METER_MMI;
4647                 if (idx[0] == 'h' && length[0] == 'h')
4648                         instr->type = INSTR_METER_HHI;
4649                 if (idx[0] == 'h' && length[0] != 'h')
4650                         instr->type = INSTR_METER_HMI;
4651                 if (idx[0] != 'h' && length[0] == 'h')
4652                         instr->type = INSTR_METER_MHI;
4653
4654                 instr->meter.metarray_id = m->id;
4655
4656                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4657                 instr->meter.idx.n_bits = fidx->n_bits;
4658                 instr->meter.idx.offset = fidx->offset / 8;
4659
4660                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4661                 instr->meter.length.n_bits = flength->n_bits;
4662                 instr->meter.length.offset = flength->offset / 8;
4663
4664                 instr->meter.color_in_val = color_in_val;
4665
4666                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4667                 instr->meter.color_out.n_bits = fcout->n_bits;
4668                 instr->meter.color_out.offset = fcout->offset / 8;
4669
4670                 return 0;
4671         }
4672
4673         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4674         if (!fidx && fcin) {
4675                 uint32_t idx_val;
4676
4677                 idx_val = strtoul(idx, &idx, 0);
4678                 CHECK(!idx[0], EINVAL);
4679
4680                 CHECK(!fcin->var_size, EINVAL);
4681
4682                 instr->type = INSTR_METER_IMM;
4683                 if (length[0] == 'h')
4684                         instr->type = INSTR_METER_IHM;
4685
4686                 instr->meter.metarray_id = m->id;
4687
4688                 instr->meter.idx_val = idx_val;
4689
4690                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4691                 instr->meter.length.n_bits = flength->n_bits;
4692                 instr->meter.length.offset = flength->offset / 8;
4693
4694                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4695                 instr->meter.color_in.n_bits = fcin->n_bits;
4696                 instr->meter.color_in.offset = fcin->offset / 8;
4697
4698                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4699                 instr->meter.color_out.n_bits = fcout->n_bits;
4700                 instr->meter.color_out.offset = fcout->offset / 8;
4701
4702                 return 0;
4703         }
4704
4705         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4706         if (!fidx && !fcin) {
4707                 uint32_t idx_val, color_in_val;
4708
4709                 idx_val = strtoul(idx, &idx, 0);
4710                 CHECK(!idx[0], EINVAL);
4711
4712                 color_in_val = strtoul(color_in, &color_in, 0);
4713                 CHECK(!color_in[0], EINVAL);
4714
4715                 instr->type = INSTR_METER_IMI;
4716                 if (length[0] == 'h')
4717                         instr->type = INSTR_METER_IHI;
4718
4719                 instr->meter.metarray_id = m->id;
4720
4721                 instr->meter.idx_val = idx_val;
4722
4723                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4724                 instr->meter.length.n_bits = flength->n_bits;
4725                 instr->meter.length.offset = flength->offset / 8;
4726
4727                 instr->meter.color_in_val = color_in_val;
4728
4729                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4730                 instr->meter.color_out.n_bits = fcout->n_bits;
4731                 instr->meter.color_out.offset = fcout->offset / 8;
4732
4733                 return 0;
4734         }
4735
4736         CHECK(0, EINVAL);
4737 }
4738
4739 static inline struct meter *
4740 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4741 {
4742         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4743
4744         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4745         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4746         uint64_t idx64 = *idx64_ptr;
4747         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
4748         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4749
4750         return &r->metarray[idx];
4751 }
4752
4753 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4754
4755 static inline struct meter *
4756 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4757 {
4758         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4759
4760         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4761         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4762         uint64_t idx64 = *idx64_ptr;
4763         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
4764
4765         return &r->metarray[idx];
4766 }
4767
4768 #else
4769
4770 #define instr_meter_idx_nbo instr_meter_idx_hbo
4771
4772 #endif
4773
4774 static inline struct meter *
4775 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4776 {
4777         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4778
4779         uint64_t idx =  ip->meter.idx_val & r->size_mask;
4780
4781         return &r->metarray[idx];
4782 }
4783
4784 static inline uint32_t
4785 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
4786 {
4787         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4788         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4789         uint64_t src64 = *src64_ptr;
4790         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
4791         uint64_t src = src64 & src64_mask;
4792
4793         return (uint32_t)src;
4794 }
4795
4796 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4797
4798 static inline uint32_t
4799 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
4800 {
4801         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4802         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4803         uint64_t src64 = *src64_ptr;
4804         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
4805
4806         return (uint32_t)src;
4807 }
4808
4809 #else
4810
4811 #define instr_meter_length_nbo instr_meter_length_hbo
4812
4813 #endif
4814
4815 static inline enum rte_color
4816 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
4817 {
4818         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
4819         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
4820         uint64_t src64 = *src64_ptr;
4821         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
4822         uint64_t src = src64 & src64_mask;
4823
4824         return (enum rte_color)src;
4825 }
4826
4827 static inline void
4828 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
4829 {
4830         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
4831         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
4832         uint64_t dst64 = *dst64_ptr;
4833         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
4834
4835         uint64_t src = (uint64_t)color_out;
4836
4837         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4838 }
4839
4840 static inline void
4841 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4842 {
4843         struct thread *t = &p->threads[p->thread_id];
4844         struct instruction *ip = t->ip;
4845         struct meter *m;
4846
4847         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
4848
4849         /* Structs. */
4850         m = instr_meter_idx_nbo(p, t, ip);
4851         rte_prefetch0(m);
4852
4853         /* Thread. */
4854         thread_ip_inc(p);
4855 }
4856
4857 static inline void
4858 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4859 {
4860         struct thread *t = &p->threads[p->thread_id];
4861         struct instruction *ip = t->ip;
4862         struct meter *m;
4863
4864         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
4865
4866         /* Structs. */
4867         m = instr_meter_idx_hbo(p, t, ip);
4868         rte_prefetch0(m);
4869
4870         /* Thread. */
4871         thread_ip_inc(p);
4872 }
4873
4874 static inline void
4875 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4876 {
4877         struct thread *t = &p->threads[p->thread_id];
4878         struct instruction *ip = t->ip;
4879         struct meter *m;
4880
4881         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
4882
4883         /* Structs. */
4884         m = instr_meter_idx_imm(p, ip);
4885         rte_prefetch0(m);
4886
4887         /* Thread. */
4888         thread_ip_inc(p);
4889 }
4890
4891 static inline void
4892 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4893 {
4894         struct thread *t = &p->threads[p->thread_id];
4895         struct instruction *ip = t->ip;
4896         struct meter *m;
4897         uint64_t time, n_pkts, n_bytes;
4898         uint32_t length;
4899         enum rte_color color_in, color_out;
4900
4901         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
4902
4903         /* Structs. */
4904         m = instr_meter_idx_nbo(p, t, ip);
4905         rte_prefetch0(m->n_pkts);
4906         time = rte_get_tsc_cycles();
4907         length = instr_meter_length_nbo(t, ip);
4908         color_in = instr_meter_color_in_hbo(t, ip);
4909
4910         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4911                 &m->profile->profile,
4912                 time,
4913                 length,
4914                 color_in);
4915
4916         color_out &= m->color_mask;
4917
4918         n_pkts = m->n_pkts[color_out];
4919         n_bytes = m->n_bytes[color_out];
4920
4921         instr_meter_color_out_hbo_set(t, ip, color_out);
4922
4923         m->n_pkts[color_out] = n_pkts + 1;
4924         m->n_bytes[color_out] = n_bytes + length;
4925
4926         /* Thread. */
4927         thread_ip_inc(p);
4928 }
4929
4930 static inline void
4931 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4932 {
4933         struct thread *t = &p->threads[p->thread_id];
4934         struct instruction *ip = t->ip;
4935         struct meter *m;
4936         uint64_t time, n_pkts, n_bytes;
4937         uint32_t length;
4938         enum rte_color color_in, color_out;
4939
4940         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
4941
4942         /* Structs. */
4943         m = instr_meter_idx_nbo(p, t, ip);
4944         rte_prefetch0(m->n_pkts);
4945         time = rte_get_tsc_cycles();
4946         length = instr_meter_length_nbo(t, ip);
4947         color_in = (enum rte_color)ip->meter.color_in_val;
4948
4949         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4950                 &m->profile->profile,
4951                 time,
4952                 length,
4953                 color_in);
4954
4955         color_out &= m->color_mask;
4956
4957         n_pkts = m->n_pkts[color_out];
4958         n_bytes = m->n_bytes[color_out];
4959
4960         instr_meter_color_out_hbo_set(t, ip, color_out);
4961
4962         m->n_pkts[color_out] = n_pkts + 1;
4963         m->n_bytes[color_out] = n_bytes + length;
4964
4965         /* Thread. */
4966         thread_ip_inc(p);
4967 }
4968
4969 static inline void
4970 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4971 {
4972         struct thread *t = &p->threads[p->thread_id];
4973         struct instruction *ip = t->ip;
4974         struct meter *m;
4975         uint64_t time, n_pkts, n_bytes;
4976         uint32_t length;
4977         enum rte_color color_in, color_out;
4978
4979         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
4980
4981         /* Structs. */
4982         m = instr_meter_idx_nbo(p, t, ip);
4983         rte_prefetch0(m->n_pkts);
4984         time = rte_get_tsc_cycles();
4985         length = instr_meter_length_hbo(t, ip);
4986         color_in = instr_meter_color_in_hbo(t, ip);
4987
4988         color_out = rte_meter_trtcm_color_aware_check(&m->m,
4989                 &m->profile->profile,
4990                 time,
4991                 length,
4992                 color_in);
4993
4994         color_out &= m->color_mask;
4995
4996         n_pkts = m->n_pkts[color_out];
4997         n_bytes = m->n_bytes[color_out];
4998
4999         instr_meter_color_out_hbo_set(t, ip, color_out);
5000
5001         m->n_pkts[color_out] = n_pkts + 1;
5002         m->n_bytes[color_out] = n_bytes + length;
5003
5004         /* Thread. */
5005         thread_ip_inc(p);
5006 }
5007 static inline void
5008 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5009 {
5010         struct thread *t = &p->threads[p->thread_id];
5011         struct instruction *ip = t->ip;
5012         struct meter *m;
5013         uint64_t time, n_pkts, n_bytes;
5014         uint32_t length;
5015         enum rte_color color_in, color_out;
5016
5017         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5018
5019         /* Structs. */
5020         m = instr_meter_idx_nbo(p, t, ip);
5021         rte_prefetch0(m->n_pkts);
5022         time = rte_get_tsc_cycles();
5023         length = instr_meter_length_hbo(t, ip);
5024         color_in = (enum rte_color)ip->meter.color_in_val;
5025
5026         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5027                 &m->profile->profile,
5028                 time,
5029                 length,
5030                 color_in);
5031
5032         color_out &= m->color_mask;
5033
5034         n_pkts = m->n_pkts[color_out];
5035         n_bytes = m->n_bytes[color_out];
5036
5037         instr_meter_color_out_hbo_set(t, ip, color_out);
5038
5039         m->n_pkts[color_out] = n_pkts + 1;
5040         m->n_bytes[color_out] = n_bytes + length;
5041
5042         /* Thread. */
5043         thread_ip_inc(p);
5044 }
5045
5046 static inline void
5047 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5048 {
5049         struct thread *t = &p->threads[p->thread_id];
5050         struct instruction *ip = t->ip;
5051         struct meter *m;
5052         uint64_t time, n_pkts, n_bytes;
5053         uint32_t length;
5054         enum rte_color color_in, color_out;
5055
5056         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5057
5058         /* Structs. */
5059         m = instr_meter_idx_hbo(p, t, ip);
5060         rte_prefetch0(m->n_pkts);
5061         time = rte_get_tsc_cycles();
5062         length = instr_meter_length_nbo(t, ip);
5063         color_in = instr_meter_color_in_hbo(t, ip);
5064
5065         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5066                 &m->profile->profile,
5067                 time,
5068                 length,
5069                 color_in);
5070
5071         color_out &= m->color_mask;
5072
5073         n_pkts = m->n_pkts[color_out];
5074         n_bytes = m->n_bytes[color_out];
5075
5076         instr_meter_color_out_hbo_set(t, ip, color_out);
5077
5078         m->n_pkts[color_out] = n_pkts + 1;
5079         m->n_bytes[color_out] = n_bytes + length;
5080
5081         /* Thread. */
5082         thread_ip_inc(p);
5083 }
5084
5085 static inline void
5086 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5087 {
5088         struct thread *t = &p->threads[p->thread_id];
5089         struct instruction *ip = t->ip;
5090         struct meter *m;
5091         uint64_t time, n_pkts, n_bytes;
5092         uint32_t length;
5093         enum rte_color color_in, color_out;
5094
5095         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5096
5097         /* Structs. */
5098         m = instr_meter_idx_hbo(p, t, ip);
5099         rte_prefetch0(m->n_pkts);
5100         time = rte_get_tsc_cycles();
5101         length = instr_meter_length_nbo(t, ip);
5102         color_in = (enum rte_color)ip->meter.color_in_val;
5103
5104         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5105                 &m->profile->profile,
5106                 time,
5107                 length,
5108                 color_in);
5109
5110         color_out &= m->color_mask;
5111
5112         n_pkts = m->n_pkts[color_out];
5113         n_bytes = m->n_bytes[color_out];
5114
5115         instr_meter_color_out_hbo_set(t, ip, color_out);
5116
5117         m->n_pkts[color_out] = n_pkts + 1;
5118         m->n_bytes[color_out] = n_bytes + length;
5119
5120         /* Thread. */
5121         thread_ip_inc(p);
5122 }
5123
5124 static inline void
5125 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5126 {
5127         struct thread *t = &p->threads[p->thread_id];
5128         struct instruction *ip = t->ip;
5129         struct meter *m;
5130         uint64_t time, n_pkts, n_bytes;
5131         uint32_t length;
5132         enum rte_color color_in, color_out;
5133
5134         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5135
5136         /* Structs. */
5137         m = instr_meter_idx_hbo(p, t, ip);
5138         rte_prefetch0(m->n_pkts);
5139         time = rte_get_tsc_cycles();
5140         length = instr_meter_length_hbo(t, ip);
5141         color_in = instr_meter_color_in_hbo(t, ip);
5142
5143         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5144                 &m->profile->profile,
5145                 time,
5146                 length,
5147                 color_in);
5148
5149         color_out &= m->color_mask;
5150
5151         n_pkts = m->n_pkts[color_out];
5152         n_bytes = m->n_bytes[color_out];
5153
5154         instr_meter_color_out_hbo_set(t, ip, color_out);
5155
5156         m->n_pkts[color_out] = n_pkts + 1;
5157         m->n_bytes[color_out] = n_bytes + length;
5158
5159         /* Thread. */
5160         thread_ip_inc(p);
5161 }
5162
5163 static inline void
5164 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5165 {
5166         struct thread *t = &p->threads[p->thread_id];
5167         struct instruction *ip = t->ip;
5168         struct meter *m;
5169         uint64_t time, n_pkts, n_bytes;
5170         uint32_t length;
5171         enum rte_color color_in, color_out;
5172
5173         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5174
5175         /* Structs. */
5176         m = instr_meter_idx_hbo(p, t, ip);
5177         rte_prefetch0(m->n_pkts);
5178         time = rte_get_tsc_cycles();
5179         length = instr_meter_length_hbo(t, ip);
5180         color_in = (enum rte_color)ip->meter.color_in_val;
5181
5182         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5183                 &m->profile->profile,
5184                 time,
5185                 length,
5186                 color_in);
5187
5188         color_out &= m->color_mask;
5189
5190         n_pkts = m->n_pkts[color_out];
5191         n_bytes = m->n_bytes[color_out];
5192
5193         instr_meter_color_out_hbo_set(t, ip, color_out);
5194
5195         m->n_pkts[color_out] = n_pkts + 1;
5196         m->n_bytes[color_out] = n_bytes + length;
5197
5198         /* Thread. */
5199         thread_ip_inc(p);
5200 }
5201
5202 static inline void
5203 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5204 {
5205         struct thread *t = &p->threads[p->thread_id];
5206         struct instruction *ip = t->ip;
5207         struct meter *m;
5208         uint64_t time, n_pkts, n_bytes;
5209         uint32_t length;
5210         enum rte_color color_in, color_out;
5211
5212         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5213
5214         /* Structs. */
5215         m = instr_meter_idx_imm(p, ip);
5216         rte_prefetch0(m->n_pkts);
5217         time = rte_get_tsc_cycles();
5218         length = instr_meter_length_nbo(t, ip);
5219         color_in = instr_meter_color_in_hbo(t, ip);
5220
5221         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5222                 &m->profile->profile,
5223                 time,
5224                 length,
5225                 color_in);
5226
5227         color_out &= m->color_mask;
5228
5229         n_pkts = m->n_pkts[color_out];
5230         n_bytes = m->n_bytes[color_out];
5231
5232         instr_meter_color_out_hbo_set(t, ip, color_out);
5233
5234         m->n_pkts[color_out] = n_pkts + 1;
5235         m->n_bytes[color_out] = n_bytes + length;
5236
5237         /* Thread. */
5238         thread_ip_inc(p);
5239 }
5240
5241 static inline void
5242 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5243 {
5244         struct thread *t = &p->threads[p->thread_id];
5245         struct instruction *ip = t->ip;
5246         struct meter *m;
5247         uint64_t time, n_pkts, n_bytes;
5248         uint32_t length;
5249         enum rte_color color_in, color_out;
5250
5251         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5252
5253         /* Structs. */
5254         m = instr_meter_idx_imm(p, ip);
5255         rte_prefetch0(m->n_pkts);
5256         time = rte_get_tsc_cycles();
5257         length = instr_meter_length_nbo(t, ip);
5258         color_in = (enum rte_color)ip->meter.color_in_val;
5259
5260         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5261                 &m->profile->profile,
5262                 time,
5263                 length,
5264                 color_in);
5265
5266         color_out &= m->color_mask;
5267
5268         n_pkts = m->n_pkts[color_out];
5269         n_bytes = m->n_bytes[color_out];
5270
5271         instr_meter_color_out_hbo_set(t, ip, color_out);
5272
5273         m->n_pkts[color_out] = n_pkts + 1;
5274         m->n_bytes[color_out] = n_bytes + length;
5275
5276         /* Thread. */
5277         thread_ip_inc(p);
5278 }
5279
5280 static inline void
5281 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5282 {
5283         struct thread *t = &p->threads[p->thread_id];
5284         struct instruction *ip = t->ip;
5285         struct meter *m;
5286         uint64_t time, n_pkts, n_bytes;
5287         uint32_t length;
5288         enum rte_color color_in, color_out;
5289
5290         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5291
5292         /* Structs. */
5293         m = instr_meter_idx_imm(p, ip);
5294         rte_prefetch0(m->n_pkts);
5295         time = rte_get_tsc_cycles();
5296         length = instr_meter_length_hbo(t, ip);
5297         color_in = instr_meter_color_in_hbo(t, ip);
5298
5299         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5300                 &m->profile->profile,
5301                 time,
5302                 length,
5303                 color_in);
5304
5305         color_out &= m->color_mask;
5306
5307         n_pkts = m->n_pkts[color_out];
5308         n_bytes = m->n_bytes[color_out];
5309
5310         instr_meter_color_out_hbo_set(t, ip, color_out);
5311
5312         m->n_pkts[color_out] = n_pkts + 1;
5313         m->n_bytes[color_out] = n_bytes + length;
5314
5315         /* Thread. */
5316         thread_ip_inc(p);
5317 }
5318 static inline void
5319 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5320 {
5321         struct thread *t = &p->threads[p->thread_id];
5322         struct instruction *ip = t->ip;
5323         struct meter *m;
5324         uint64_t time, n_pkts, n_bytes;
5325         uint32_t length;
5326         enum rte_color color_in, color_out;
5327
5328         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5329
5330         /* Structs. */
5331         m = instr_meter_idx_imm(p, ip);
5332         rte_prefetch0(m->n_pkts);
5333         time = rte_get_tsc_cycles();
5334         length = instr_meter_length_hbo(t, ip);
5335         color_in = (enum rte_color)ip->meter.color_in_val;
5336
5337         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5338                 &m->profile->profile,
5339                 time,
5340                 length,
5341                 color_in);
5342
5343         color_out &= m->color_mask;
5344
5345         n_pkts = m->n_pkts[color_out];
5346         n_bytes = m->n_bytes[color_out];
5347
5348         instr_meter_color_out_hbo_set(t, ip, color_out);
5349
5350         m->n_pkts[color_out] = n_pkts + 1;
5351         m->n_bytes[color_out] = n_bytes + length;
5352
5353         /* Thread. */
5354         thread_ip_inc(p);
5355 }
5356
5357 /*
5358  * jmp.
5359  */
5360 static int
5361 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5362                     struct action *action __rte_unused,
5363                     char **tokens,
5364                     int n_tokens,
5365                     struct instruction *instr,
5366                     struct instruction_data *data)
5367 {
5368         CHECK(n_tokens == 2, EINVAL);
5369
5370         strcpy(data->jmp_label, tokens[1]);
5371
5372         instr->type = INSTR_JMP;
5373         instr->jmp.ip = NULL; /* Resolved later. */
5374         return 0;
5375 }
5376
5377 static int
5378 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5379                           struct action *action __rte_unused,
5380                           char **tokens,
5381                           int n_tokens,
5382                           struct instruction *instr,
5383                           struct instruction_data *data)
5384 {
5385         struct header *h;
5386
5387         CHECK(n_tokens == 3, EINVAL);
5388
5389         strcpy(data->jmp_label, tokens[1]);
5390
5391         h = header_parse(p, tokens[2]);
5392         CHECK(h, EINVAL);
5393
5394         instr->type = INSTR_JMP_VALID;
5395         instr->jmp.ip = NULL; /* Resolved later. */
5396         instr->jmp.header_id = h->id;
5397         return 0;
5398 }
5399
5400 static int
5401 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5402                             struct action *action __rte_unused,
5403                             char **tokens,
5404                             int n_tokens,
5405                             struct instruction *instr,
5406                             struct instruction_data *data)
5407 {
5408         struct header *h;
5409
5410         CHECK(n_tokens == 3, EINVAL);
5411
5412         strcpy(data->jmp_label, tokens[1]);
5413
5414         h = header_parse(p, tokens[2]);
5415         CHECK(h, EINVAL);
5416
5417         instr->type = INSTR_JMP_INVALID;
5418         instr->jmp.ip = NULL; /* Resolved later. */
5419         instr->jmp.header_id = h->id;
5420         return 0;
5421 }
5422
5423 static int
5424 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5425                         struct action *action,
5426                         char **tokens,
5427                         int n_tokens,
5428                         struct instruction *instr,
5429                         struct instruction_data *data)
5430 {
5431         CHECK(!action, EINVAL);
5432         CHECK(n_tokens == 2, EINVAL);
5433
5434         strcpy(data->jmp_label, tokens[1]);
5435
5436         instr->type = INSTR_JMP_HIT;
5437         instr->jmp.ip = NULL; /* Resolved later. */
5438         return 0;
5439 }
5440
5441 static int
5442 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5443                          struct action *action,
5444                          char **tokens,
5445                          int n_tokens,
5446                          struct instruction *instr,
5447                          struct instruction_data *data)
5448 {
5449         CHECK(!action, EINVAL);
5450         CHECK(n_tokens == 2, EINVAL);
5451
5452         strcpy(data->jmp_label, tokens[1]);
5453
5454         instr->type = INSTR_JMP_MISS;
5455         instr->jmp.ip = NULL; /* Resolved later. */
5456         return 0;
5457 }
5458
5459 static int
5460 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5461                                struct action *action,
5462                                char **tokens,
5463                                int n_tokens,
5464                                struct instruction *instr,
5465                                struct instruction_data *data)
5466 {
5467         struct action *a;
5468
5469         CHECK(!action, EINVAL);
5470         CHECK(n_tokens == 3, EINVAL);
5471
5472         strcpy(data->jmp_label, tokens[1]);
5473
5474         a = action_find(p, tokens[2]);
5475         CHECK(a, EINVAL);
5476
5477         instr->type = INSTR_JMP_ACTION_HIT;
5478         instr->jmp.ip = NULL; /* Resolved later. */
5479         instr->jmp.action_id = a->id;
5480         return 0;
5481 }
5482
5483 static int
5484 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5485                                 struct action *action,
5486                                 char **tokens,
5487                                 int n_tokens,
5488                                 struct instruction *instr,
5489                                 struct instruction_data *data)
5490 {
5491         struct action *a;
5492
5493         CHECK(!action, EINVAL);
5494         CHECK(n_tokens == 3, EINVAL);
5495
5496         strcpy(data->jmp_label, tokens[1]);
5497
5498         a = action_find(p, tokens[2]);
5499         CHECK(a, EINVAL);
5500
5501         instr->type = INSTR_JMP_ACTION_MISS;
5502         instr->jmp.ip = NULL; /* Resolved later. */
5503         instr->jmp.action_id = a->id;
5504         return 0;
5505 }
5506
5507 static int
5508 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5509                        struct action *action,
5510                        char **tokens,
5511                        int n_tokens,
5512                        struct instruction *instr,
5513                        struct instruction_data *data)
5514 {
5515         char *a = tokens[2], *b = tokens[3];
5516         struct field *fa, *fb;
5517         uint64_t b_val;
5518         uint32_t a_struct_id, b_struct_id;
5519
5520         CHECK(n_tokens == 4, EINVAL);
5521
5522         strcpy(data->jmp_label, tokens[1]);
5523
5524         fa = struct_field_parse(p, action, a, &a_struct_id);
5525         CHECK(fa, EINVAL);
5526         CHECK(!fa->var_size, EINVAL);
5527
5528         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5529         fb = struct_field_parse(p, action, b, &b_struct_id);
5530         if (fb) {
5531                 CHECK(!fb->var_size, EINVAL);
5532
5533                 instr->type = INSTR_JMP_EQ;
5534                 if (a[0] != 'h' && b[0] == 'h')
5535                         instr->type = INSTR_JMP_EQ_MH;
5536                 if (a[0] == 'h' && b[0] != 'h')
5537                         instr->type = INSTR_JMP_EQ_HM;
5538                 if (a[0] == 'h' && b[0] == 'h')
5539                         instr->type = INSTR_JMP_EQ_HH;
5540                 instr->jmp.ip = NULL; /* Resolved later. */
5541
5542                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5543                 instr->jmp.a.n_bits = fa->n_bits;
5544                 instr->jmp.a.offset = fa->offset / 8;
5545                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5546                 instr->jmp.b.n_bits = fb->n_bits;
5547                 instr->jmp.b.offset = fb->offset / 8;
5548                 return 0;
5549         }
5550
5551         /* JMP_EQ_I. */
5552         b_val = strtoull(b, &b, 0);
5553         CHECK(!b[0], EINVAL);
5554
5555         if (a[0] == 'h')
5556                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5557
5558         instr->type = INSTR_JMP_EQ_I;
5559         instr->jmp.ip = NULL; /* Resolved later. */
5560         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5561         instr->jmp.a.n_bits = fa->n_bits;
5562         instr->jmp.a.offset = fa->offset / 8;
5563         instr->jmp.b_val = b_val;
5564         return 0;
5565 }
5566
5567 static int
5568 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5569                         struct action *action,
5570                         char **tokens,
5571                         int n_tokens,
5572                         struct instruction *instr,
5573                         struct instruction_data *data)
5574 {
5575         char *a = tokens[2], *b = tokens[3];
5576         struct field *fa, *fb;
5577         uint64_t b_val;
5578         uint32_t a_struct_id, b_struct_id;
5579
5580         CHECK(n_tokens == 4, EINVAL);
5581
5582         strcpy(data->jmp_label, tokens[1]);
5583
5584         fa = struct_field_parse(p, action, a, &a_struct_id);
5585         CHECK(fa, EINVAL);
5586         CHECK(!fa->var_size, EINVAL);
5587
5588         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5589         fb = struct_field_parse(p, action, b, &b_struct_id);
5590         if (fb) {
5591                 CHECK(!fb->var_size, EINVAL);
5592
5593                 instr->type = INSTR_JMP_NEQ;
5594                 if (a[0] != 'h' && b[0] == 'h')
5595                         instr->type = INSTR_JMP_NEQ_MH;
5596                 if (a[0] == 'h' && b[0] != 'h')
5597                         instr->type = INSTR_JMP_NEQ_HM;
5598                 if (a[0] == 'h' && b[0] == 'h')
5599                         instr->type = INSTR_JMP_NEQ_HH;
5600                 instr->jmp.ip = NULL; /* Resolved later. */
5601
5602                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5603                 instr->jmp.a.n_bits = fa->n_bits;
5604                 instr->jmp.a.offset = fa->offset / 8;
5605                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5606                 instr->jmp.b.n_bits = fb->n_bits;
5607                 instr->jmp.b.offset = fb->offset / 8;
5608                 return 0;
5609         }
5610
5611         /* JMP_NEQ_I. */
5612         b_val = strtoull(b, &b, 0);
5613         CHECK(!b[0], EINVAL);
5614
5615         if (a[0] == 'h')
5616                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5617
5618         instr->type = INSTR_JMP_NEQ_I;
5619         instr->jmp.ip = NULL; /* Resolved later. */
5620         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5621         instr->jmp.a.n_bits = fa->n_bits;
5622         instr->jmp.a.offset = fa->offset / 8;
5623         instr->jmp.b_val = b_val;
5624         return 0;
5625 }
5626
5627 static int
5628 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5629                        struct action *action,
5630                        char **tokens,
5631                        int n_tokens,
5632                        struct instruction *instr,
5633                        struct instruction_data *data)
5634 {
5635         char *a = tokens[2], *b = tokens[3];
5636         struct field *fa, *fb;
5637         uint64_t b_val;
5638         uint32_t a_struct_id, b_struct_id;
5639
5640         CHECK(n_tokens == 4, EINVAL);
5641
5642         strcpy(data->jmp_label, tokens[1]);
5643
5644         fa = struct_field_parse(p, action, a, &a_struct_id);
5645         CHECK(fa, EINVAL);
5646         CHECK(!fa->var_size, EINVAL);
5647
5648         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5649         fb = struct_field_parse(p, action, b, &b_struct_id);
5650         if (fb) {
5651                 CHECK(!fb->var_size, EINVAL);
5652
5653                 instr->type = INSTR_JMP_LT;
5654                 if (a[0] == 'h' && b[0] != 'h')
5655                         instr->type = INSTR_JMP_LT_HM;
5656                 if (a[0] != 'h' && b[0] == 'h')
5657                         instr->type = INSTR_JMP_LT_MH;
5658                 if (a[0] == 'h' && b[0] == 'h')
5659                         instr->type = INSTR_JMP_LT_HH;
5660                 instr->jmp.ip = NULL; /* Resolved later. */
5661
5662                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5663                 instr->jmp.a.n_bits = fa->n_bits;
5664                 instr->jmp.a.offset = fa->offset / 8;
5665                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5666                 instr->jmp.b.n_bits = fb->n_bits;
5667                 instr->jmp.b.offset = fb->offset / 8;
5668                 return 0;
5669         }
5670
5671         /* JMP_LT_MI, JMP_LT_HI. */
5672         b_val = strtoull(b, &b, 0);
5673         CHECK(!b[0], EINVAL);
5674
5675         instr->type = INSTR_JMP_LT_MI;
5676         if (a[0] == 'h')
5677                 instr->type = INSTR_JMP_LT_HI;
5678         instr->jmp.ip = NULL; /* Resolved later. */
5679
5680         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5681         instr->jmp.a.n_bits = fa->n_bits;
5682         instr->jmp.a.offset = fa->offset / 8;
5683         instr->jmp.b_val = b_val;
5684         return 0;
5685 }
5686
5687 static int
5688 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5689                        struct action *action,
5690                        char **tokens,
5691                        int n_tokens,
5692                        struct instruction *instr,
5693                        struct instruction_data *data)
5694 {
5695         char *a = tokens[2], *b = tokens[3];
5696         struct field *fa, *fb;
5697         uint64_t b_val;
5698         uint32_t a_struct_id, b_struct_id;
5699
5700         CHECK(n_tokens == 4, EINVAL);
5701
5702         strcpy(data->jmp_label, tokens[1]);
5703
5704         fa = struct_field_parse(p, action, a, &a_struct_id);
5705         CHECK(fa, EINVAL);
5706         CHECK(!fa->var_size, EINVAL);
5707
5708         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5709         fb = struct_field_parse(p, action, b, &b_struct_id);
5710         if (fb) {
5711                 CHECK(!fb->var_size, EINVAL);
5712
5713                 instr->type = INSTR_JMP_GT;
5714                 if (a[0] == 'h' && b[0] != 'h')
5715                         instr->type = INSTR_JMP_GT_HM;
5716                 if (a[0] != 'h' && b[0] == 'h')
5717                         instr->type = INSTR_JMP_GT_MH;
5718                 if (a[0] == 'h' && b[0] == 'h')
5719                         instr->type = INSTR_JMP_GT_HH;
5720                 instr->jmp.ip = NULL; /* Resolved later. */
5721
5722                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5723                 instr->jmp.a.n_bits = fa->n_bits;
5724                 instr->jmp.a.offset = fa->offset / 8;
5725                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5726                 instr->jmp.b.n_bits = fb->n_bits;
5727                 instr->jmp.b.offset = fb->offset / 8;
5728                 return 0;
5729         }
5730
5731         /* JMP_GT_MI, JMP_GT_HI. */
5732         b_val = strtoull(b, &b, 0);
5733         CHECK(!b[0], EINVAL);
5734
5735         instr->type = INSTR_JMP_GT_MI;
5736         if (a[0] == 'h')
5737                 instr->type = INSTR_JMP_GT_HI;
5738         instr->jmp.ip = NULL; /* Resolved later. */
5739
5740         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5741         instr->jmp.a.n_bits = fa->n_bits;
5742         instr->jmp.a.offset = fa->offset / 8;
5743         instr->jmp.b_val = b_val;
5744         return 0;
5745 }
5746
5747 static inline void
5748 instr_jmp_exec(struct rte_swx_pipeline *p)
5749 {
5750         struct thread *t = &p->threads[p->thread_id];
5751         struct instruction *ip = t->ip;
5752
5753         TRACE("[Thread %2u] jmp\n", p->thread_id);
5754
5755         thread_ip_set(t, ip->jmp.ip);
5756 }
5757
5758 static inline void
5759 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5760 {
5761         struct thread *t = &p->threads[p->thread_id];
5762         struct instruction *ip = t->ip;
5763         uint32_t header_id = ip->jmp.header_id;
5764
5765         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5766
5767         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5768 }
5769
5770 static inline void
5771 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5772 {
5773         struct thread *t = &p->threads[p->thread_id];
5774         struct instruction *ip = t->ip;
5775         uint32_t header_id = ip->jmp.header_id;
5776
5777         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5778
5779         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5780 }
5781
5782 static inline void
5783 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5784 {
5785         struct thread *t = &p->threads[p->thread_id];
5786         struct instruction *ip = t->ip;
5787         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5788
5789         TRACE("[Thread %2u] jmph\n", p->thread_id);
5790
5791         t->ip = ip_next[t->hit];
5792 }
5793
5794 static inline void
5795 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5796 {
5797         struct thread *t = &p->threads[p->thread_id];
5798         struct instruction *ip = t->ip;
5799         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5800
5801         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5802
5803         t->ip = ip_next[t->hit];
5804 }
5805
5806 static inline void
5807 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5808 {
5809         struct thread *t = &p->threads[p->thread_id];
5810         struct instruction *ip = t->ip;
5811
5812         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5813
5814         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5815 }
5816
5817 static inline void
5818 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5819 {
5820         struct thread *t = &p->threads[p->thread_id];
5821         struct instruction *ip = t->ip;
5822
5823         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5824
5825         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5826 }
5827
5828 static inline void
5829 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5830 {
5831         struct thread *t = &p->threads[p->thread_id];
5832         struct instruction *ip = t->ip;
5833
5834         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5835
5836         JMP_CMP(t, ip, ==);
5837 }
5838
5839 static inline void
5840 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5841 {
5842         struct thread *t = &p->threads[p->thread_id];
5843         struct instruction *ip = t->ip;
5844
5845         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5846
5847         JMP_CMP_MH(t, ip, ==);
5848 }
5849
5850 static inline void
5851 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5852 {
5853         struct thread *t = &p->threads[p->thread_id];
5854         struct instruction *ip = t->ip;
5855
5856         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5857
5858         JMP_CMP_HM(t, ip, ==);
5859 }
5860
5861 static inline void
5862 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5863 {
5864         struct thread *t = &p->threads[p->thread_id];
5865         struct instruction *ip = t->ip;
5866
5867         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5868
5869         JMP_CMP_HH_FAST(t, ip, ==);
5870 }
5871
5872 static inline void
5873 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5874 {
5875         struct thread *t = &p->threads[p->thread_id];
5876         struct instruction *ip = t->ip;
5877
5878         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5879
5880         JMP_CMP_I(t, ip, ==);
5881 }
5882
5883 static inline void
5884 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5885 {
5886         struct thread *t = &p->threads[p->thread_id];
5887         struct instruction *ip = t->ip;
5888
5889         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5890
5891         JMP_CMP(t, ip, !=);
5892 }
5893
5894 static inline void
5895 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5896 {
5897         struct thread *t = &p->threads[p->thread_id];
5898         struct instruction *ip = t->ip;
5899
5900         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5901
5902         JMP_CMP_MH(t, ip, !=);
5903 }
5904
5905 static inline void
5906 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5907 {
5908         struct thread *t = &p->threads[p->thread_id];
5909         struct instruction *ip = t->ip;
5910
5911         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5912
5913         JMP_CMP_HM(t, ip, !=);
5914 }
5915
5916 static inline void
5917 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5918 {
5919         struct thread *t = &p->threads[p->thread_id];
5920         struct instruction *ip = t->ip;
5921
5922         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5923
5924         JMP_CMP_HH_FAST(t, ip, !=);
5925 }
5926
5927 static inline void
5928 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5929 {
5930         struct thread *t = &p->threads[p->thread_id];
5931         struct instruction *ip = t->ip;
5932
5933         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5934
5935         JMP_CMP_I(t, ip, !=);
5936 }
5937
5938 static inline void
5939 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5940 {
5941         struct thread *t = &p->threads[p->thread_id];
5942         struct instruction *ip = t->ip;
5943
5944         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5945
5946         JMP_CMP(t, ip, <);
5947 }
5948
5949 static inline void
5950 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5951 {
5952         struct thread *t = &p->threads[p->thread_id];
5953         struct instruction *ip = t->ip;
5954
5955         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5956
5957         JMP_CMP_MH(t, ip, <);
5958 }
5959
5960 static inline void
5961 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5962 {
5963         struct thread *t = &p->threads[p->thread_id];
5964         struct instruction *ip = t->ip;
5965
5966         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5967
5968         JMP_CMP_HM(t, ip, <);
5969 }
5970
5971 static inline void
5972 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5973 {
5974         struct thread *t = &p->threads[p->thread_id];
5975         struct instruction *ip = t->ip;
5976
5977         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5978
5979         JMP_CMP_HH(t, ip, <);
5980 }
5981
5982 static inline void
5983 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5984 {
5985         struct thread *t = &p->threads[p->thread_id];
5986         struct instruction *ip = t->ip;
5987
5988         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5989
5990         JMP_CMP_MI(t, ip, <);
5991 }
5992
5993 static inline void
5994 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5995 {
5996         struct thread *t = &p->threads[p->thread_id];
5997         struct instruction *ip = t->ip;
5998
5999         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6000
6001         JMP_CMP_HI(t, ip, <);
6002 }
6003
6004 static inline void
6005 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6006 {
6007         struct thread *t = &p->threads[p->thread_id];
6008         struct instruction *ip = t->ip;
6009
6010         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6011
6012         JMP_CMP(t, ip, >);
6013 }
6014
6015 static inline void
6016 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6017 {
6018         struct thread *t = &p->threads[p->thread_id];
6019         struct instruction *ip = t->ip;
6020
6021         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6022
6023         JMP_CMP_MH(t, ip, >);
6024 }
6025
6026 static inline void
6027 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6028 {
6029         struct thread *t = &p->threads[p->thread_id];
6030         struct instruction *ip = t->ip;
6031
6032         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6033
6034         JMP_CMP_HM(t, ip, >);
6035 }
6036
6037 static inline void
6038 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6039 {
6040         struct thread *t = &p->threads[p->thread_id];
6041         struct instruction *ip = t->ip;
6042
6043         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6044
6045         JMP_CMP_HH(t, ip, >);
6046 }
6047
6048 static inline void
6049 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6050 {
6051         struct thread *t = &p->threads[p->thread_id];
6052         struct instruction *ip = t->ip;
6053
6054         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6055
6056         JMP_CMP_MI(t, ip, >);
6057 }
6058
6059 static inline void
6060 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6061 {
6062         struct thread *t = &p->threads[p->thread_id];
6063         struct instruction *ip = t->ip;
6064
6065         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6066
6067         JMP_CMP_HI(t, ip, >);
6068 }
6069
6070 /*
6071  * return.
6072  */
6073 static int
6074 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6075                        struct action *action,
6076                        char **tokens __rte_unused,
6077                        int n_tokens,
6078                        struct instruction *instr,
6079                        struct instruction_data *data __rte_unused)
6080 {
6081         CHECK(action, EINVAL);
6082         CHECK(n_tokens == 1, EINVAL);
6083
6084         instr->type = INSTR_RETURN;
6085         return 0;
6086 }
6087
6088 static inline void
6089 instr_return_exec(struct rte_swx_pipeline *p)
6090 {
6091         struct thread *t = &p->threads[p->thread_id];
6092
6093         TRACE("[Thread %2u] return\n", p->thread_id);
6094
6095         t->ip = t->ret;
6096 }
6097
6098 static int
6099 instr_translate(struct rte_swx_pipeline *p,
6100                 struct action *action,
6101                 char *string,
6102                 struct instruction *instr,
6103                 struct instruction_data *data)
6104 {
6105         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6106         int n_tokens = 0, tpos = 0;
6107
6108         /* Parse the instruction string into tokens. */
6109         for ( ; ; ) {
6110                 char *token;
6111
6112                 token = strtok_r(string, " \t\v", &string);
6113                 if (!token)
6114                         break;
6115
6116                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6117                 CHECK_NAME(token, EINVAL);
6118
6119                 tokens[n_tokens] = token;
6120                 n_tokens++;
6121         }
6122
6123         CHECK(n_tokens, EINVAL);
6124
6125         /* Handle the optional instruction label. */
6126         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6127                 strcpy(data->label, tokens[0]);
6128
6129                 tpos += 2;
6130                 CHECK(n_tokens - tpos, EINVAL);
6131         }
6132
6133         /* Identify the instruction type. */
6134         if (!strcmp(tokens[tpos], "rx"))
6135                 return instr_rx_translate(p,
6136                                           action,
6137                                           &tokens[tpos],
6138                                           n_tokens - tpos,
6139                                           instr,
6140                                           data);
6141
6142         if (!strcmp(tokens[tpos], "tx"))
6143                 return instr_tx_translate(p,
6144                                           action,
6145                                           &tokens[tpos],
6146                                           n_tokens - tpos,
6147                                           instr,
6148                                           data);
6149
6150         if (!strcmp(tokens[tpos], "drop"))
6151                 return instr_drop_translate(p,
6152                                             action,
6153                                             &tokens[tpos],
6154                                             n_tokens - tpos,
6155                                             instr,
6156                                             data);
6157
6158         if (!strcmp(tokens[tpos], "extract"))
6159                 return instr_hdr_extract_translate(p,
6160                                                    action,
6161                                                    &tokens[tpos],
6162                                                    n_tokens - tpos,
6163                                                    instr,
6164                                                    data);
6165
6166         if (!strcmp(tokens[tpos], "lookahead"))
6167                 return instr_hdr_lookahead_translate(p,
6168                                                      action,
6169                                                      &tokens[tpos],
6170                                                      n_tokens - tpos,
6171                                                      instr,
6172                                                      data);
6173
6174         if (!strcmp(tokens[tpos], "emit"))
6175                 return instr_hdr_emit_translate(p,
6176                                                 action,
6177                                                 &tokens[tpos],
6178                                                 n_tokens - tpos,
6179                                                 instr,
6180                                                 data);
6181
6182         if (!strcmp(tokens[tpos], "validate"))
6183                 return instr_hdr_validate_translate(p,
6184                                                     action,
6185                                                     &tokens[tpos],
6186                                                     n_tokens - tpos,
6187                                                     instr,
6188                                                     data);
6189
6190         if (!strcmp(tokens[tpos], "invalidate"))
6191                 return instr_hdr_invalidate_translate(p,
6192                                                       action,
6193                                                       &tokens[tpos],
6194                                                       n_tokens - tpos,
6195                                                       instr,
6196                                                       data);
6197
6198         if (!strcmp(tokens[tpos], "mov"))
6199                 return instr_mov_translate(p,
6200                                            action,
6201                                            &tokens[tpos],
6202                                            n_tokens - tpos,
6203                                            instr,
6204                                            data);
6205
6206         if (!strcmp(tokens[tpos], "add"))
6207                 return instr_alu_add_translate(p,
6208                                                action,
6209                                                &tokens[tpos],
6210                                                n_tokens - tpos,
6211                                                instr,
6212                                                data);
6213
6214         if (!strcmp(tokens[tpos], "sub"))
6215                 return instr_alu_sub_translate(p,
6216                                                action,
6217                                                &tokens[tpos],
6218                                                n_tokens - tpos,
6219                                                instr,
6220                                                data);
6221
6222         if (!strcmp(tokens[tpos], "ckadd"))
6223                 return instr_alu_ckadd_translate(p,
6224                                                  action,
6225                                                  &tokens[tpos],
6226                                                  n_tokens - tpos,
6227                                                  instr,
6228                                                  data);
6229
6230         if (!strcmp(tokens[tpos], "cksub"))
6231                 return instr_alu_cksub_translate(p,
6232                                                  action,
6233                                                  &tokens[tpos],
6234                                                  n_tokens - tpos,
6235                                                  instr,
6236                                                  data);
6237
6238         if (!strcmp(tokens[tpos], "and"))
6239                 return instr_alu_and_translate(p,
6240                                                action,
6241                                                &tokens[tpos],
6242                                                n_tokens - tpos,
6243                                                instr,
6244                                                data);
6245
6246         if (!strcmp(tokens[tpos], "or"))
6247                 return instr_alu_or_translate(p,
6248                                               action,
6249                                               &tokens[tpos],
6250                                               n_tokens - tpos,
6251                                               instr,
6252                                               data);
6253
6254         if (!strcmp(tokens[tpos], "xor"))
6255                 return instr_alu_xor_translate(p,
6256                                                action,
6257                                                &tokens[tpos],
6258                                                n_tokens - tpos,
6259                                                instr,
6260                                                data);
6261
6262         if (!strcmp(tokens[tpos], "shl"))
6263                 return instr_alu_shl_translate(p,
6264                                                action,
6265                                                &tokens[tpos],
6266                                                n_tokens - tpos,
6267                                                instr,
6268                                                data);
6269
6270         if (!strcmp(tokens[tpos], "shr"))
6271                 return instr_alu_shr_translate(p,
6272                                                action,
6273                                                &tokens[tpos],
6274                                                n_tokens - tpos,
6275                                                instr,
6276                                                data);
6277
6278         if (!strcmp(tokens[tpos], "regprefetch"))
6279                 return instr_regprefetch_translate(p,
6280                                                    action,
6281                                                    &tokens[tpos],
6282                                                    n_tokens - tpos,
6283                                                    instr,
6284                                                    data);
6285
6286         if (!strcmp(tokens[tpos], "regrd"))
6287                 return instr_regrd_translate(p,
6288                                              action,
6289                                              &tokens[tpos],
6290                                              n_tokens - tpos,
6291                                              instr,
6292                                              data);
6293
6294         if (!strcmp(tokens[tpos], "regwr"))
6295                 return instr_regwr_translate(p,
6296                                              action,
6297                                              &tokens[tpos],
6298                                              n_tokens - tpos,
6299                                              instr,
6300                                              data);
6301
6302         if (!strcmp(tokens[tpos], "regadd"))
6303                 return instr_regadd_translate(p,
6304                                               action,
6305                                               &tokens[tpos],
6306                                               n_tokens - tpos,
6307                                               instr,
6308                                               data);
6309
6310         if (!strcmp(tokens[tpos], "metprefetch"))
6311                 return instr_metprefetch_translate(p,
6312                                                    action,
6313                                                    &tokens[tpos],
6314                                                    n_tokens - tpos,
6315                                                    instr,
6316                                                    data);
6317
6318         if (!strcmp(tokens[tpos], "meter"))
6319                 return instr_meter_translate(p,
6320                                              action,
6321                                              &tokens[tpos],
6322                                              n_tokens - tpos,
6323                                              instr,
6324                                              data);
6325
6326         if (!strcmp(tokens[tpos], "table"))
6327                 return instr_table_translate(p,
6328                                              action,
6329                                              &tokens[tpos],
6330                                              n_tokens - tpos,
6331                                              instr,
6332                                              data);
6333
6334         if (!strcmp(tokens[tpos], "learn"))
6335                 return instr_learn_translate(p,
6336                                              action,
6337                                              &tokens[tpos],
6338                                              n_tokens - tpos,
6339                                              instr,
6340                                              data);
6341
6342         if (!strcmp(tokens[tpos], "forget"))
6343                 return instr_forget_translate(p,
6344                                               action,
6345                                               &tokens[tpos],
6346                                               n_tokens - tpos,
6347                                               instr,
6348                                               data);
6349
6350         if (!strcmp(tokens[tpos], "extern"))
6351                 return instr_extern_translate(p,
6352                                               action,
6353                                               &tokens[tpos],
6354                                               n_tokens - tpos,
6355                                               instr,
6356                                               data);
6357
6358         if (!strcmp(tokens[tpos], "jmp"))
6359                 return instr_jmp_translate(p,
6360                                            action,
6361                                            &tokens[tpos],
6362                                            n_tokens - tpos,
6363                                            instr,
6364                                            data);
6365
6366         if (!strcmp(tokens[tpos], "jmpv"))
6367                 return instr_jmp_valid_translate(p,
6368                                                  action,
6369                                                  &tokens[tpos],
6370                                                  n_tokens - tpos,
6371                                                  instr,
6372                                                  data);
6373
6374         if (!strcmp(tokens[tpos], "jmpnv"))
6375                 return instr_jmp_invalid_translate(p,
6376                                                    action,
6377                                                    &tokens[tpos],
6378                                                    n_tokens - tpos,
6379                                                    instr,
6380                                                    data);
6381
6382         if (!strcmp(tokens[tpos], "jmph"))
6383                 return instr_jmp_hit_translate(p,
6384                                                action,
6385                                                &tokens[tpos],
6386                                                n_tokens - tpos,
6387                                                instr,
6388                                                data);
6389
6390         if (!strcmp(tokens[tpos], "jmpnh"))
6391                 return instr_jmp_miss_translate(p,
6392                                                 action,
6393                                                 &tokens[tpos],
6394                                                 n_tokens - tpos,
6395                                                 instr,
6396                                                 data);
6397
6398         if (!strcmp(tokens[tpos], "jmpa"))
6399                 return instr_jmp_action_hit_translate(p,
6400                                                       action,
6401                                                       &tokens[tpos],
6402                                                       n_tokens - tpos,
6403                                                       instr,
6404                                                       data);
6405
6406         if (!strcmp(tokens[tpos], "jmpna"))
6407                 return instr_jmp_action_miss_translate(p,
6408                                                        action,
6409                                                        &tokens[tpos],
6410                                                        n_tokens - tpos,
6411                                                        instr,
6412                                                        data);
6413
6414         if (!strcmp(tokens[tpos], "jmpeq"))
6415                 return instr_jmp_eq_translate(p,
6416                                               action,
6417                                               &tokens[tpos],
6418                                               n_tokens - tpos,
6419                                               instr,
6420                                               data);
6421
6422         if (!strcmp(tokens[tpos], "jmpneq"))
6423                 return instr_jmp_neq_translate(p,
6424                                                action,
6425                                                &tokens[tpos],
6426                                                n_tokens - tpos,
6427                                                instr,
6428                                                data);
6429
6430         if (!strcmp(tokens[tpos], "jmplt"))
6431                 return instr_jmp_lt_translate(p,
6432                                               action,
6433                                               &tokens[tpos],
6434                                               n_tokens - tpos,
6435                                               instr,
6436                                               data);
6437
6438         if (!strcmp(tokens[tpos], "jmpgt"))
6439                 return instr_jmp_gt_translate(p,
6440                                               action,
6441                                               &tokens[tpos],
6442                                               n_tokens - tpos,
6443                                               instr,
6444                                               data);
6445
6446         if (!strcmp(tokens[tpos], "return"))
6447                 return instr_return_translate(p,
6448                                               action,
6449                                               &tokens[tpos],
6450                                               n_tokens - tpos,
6451                                               instr,
6452                                               data);
6453
6454         CHECK(0, EINVAL);
6455 }
6456
6457 static struct instruction_data *
6458 label_find(struct instruction_data *data, uint32_t n, const char *label)
6459 {
6460         uint32_t i;
6461
6462         for (i = 0; i < n; i++)
6463                 if (!strcmp(label, data[i].label))
6464                         return &data[i];
6465
6466         return NULL;
6467 }
6468
6469 static uint32_t
6470 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6471 {
6472         uint32_t count = 0, i;
6473
6474         if (!label[0])
6475                 return 0;
6476
6477         for (i = 0; i < n; i++)
6478                 if (!strcmp(label, data[i].jmp_label))
6479                         count++;
6480
6481         return count;
6482 }
6483
6484 static int
6485 instr_label_check(struct instruction_data *instruction_data,
6486                   uint32_t n_instructions)
6487 {
6488         uint32_t i;
6489
6490         /* Check that all instruction labels are unique. */
6491         for (i = 0; i < n_instructions; i++) {
6492                 struct instruction_data *data = &instruction_data[i];
6493                 char *label = data->label;
6494                 uint32_t j;
6495
6496                 if (!label[0])
6497                         continue;
6498
6499                 for (j = i + 1; j < n_instructions; j++)
6500                         CHECK(strcmp(label, data[j].label), EINVAL);
6501         }
6502
6503         /* Get users for each instruction label. */
6504         for (i = 0; i < n_instructions; i++) {
6505                 struct instruction_data *data = &instruction_data[i];
6506                 char *label = data->label;
6507
6508                 data->n_users = label_is_used(instruction_data,
6509                                               n_instructions,
6510                                               label);
6511         }
6512
6513         return 0;
6514 }
6515
6516 static int
6517 instr_jmp_resolve(struct instruction *instructions,
6518                   struct instruction_data *instruction_data,
6519                   uint32_t n_instructions)
6520 {
6521         uint32_t i;
6522
6523         for (i = 0; i < n_instructions; i++) {
6524                 struct instruction *instr = &instructions[i];
6525                 struct instruction_data *data = &instruction_data[i];
6526                 struct instruction_data *found;
6527
6528                 if (!instruction_is_jmp(instr))
6529                         continue;
6530
6531                 found = label_find(instruction_data,
6532                                    n_instructions,
6533                                    data->jmp_label);
6534                 CHECK(found, EINVAL);
6535
6536                 instr->jmp.ip = &instructions[found - instruction_data];
6537         }
6538
6539         return 0;
6540 }
6541
6542 static int
6543 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6544              struct action *a,
6545              struct instruction *instr,
6546              struct instruction_data *data __rte_unused,
6547              uint32_t n_instructions)
6548 {
6549         if (!a) {
6550                 enum instruction_type type;
6551                 uint32_t i;
6552
6553                 /* Check that the first instruction is rx. */
6554                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6555
6556                 /* Check that there is at least one tx instruction. */
6557                 for (i = 0; i < n_instructions; i++) {
6558                         type = instr[i].type;
6559
6560                         if (instruction_is_tx(type))
6561                                 break;
6562                 }
6563                 CHECK(i < n_instructions, EINVAL);
6564
6565                 /* Check that the last instruction is either tx or unconditional
6566                  * jump.
6567                  */
6568                 type = instr[n_instructions - 1].type;
6569                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6570         }
6571
6572         if (a) {
6573                 enum instruction_type type;
6574                 uint32_t i;
6575
6576                 /* Check that there is at least one return or tx instruction. */
6577                 for (i = 0; i < n_instructions; i++) {
6578                         type = instr[i].type;
6579
6580                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6581                                 break;
6582                 }
6583                 CHECK(i < n_instructions, EINVAL);
6584         }
6585
6586         return 0;
6587 }
6588
6589 static uint32_t
6590 instr_compact(struct instruction *instructions,
6591               struct instruction_data *instruction_data,
6592               uint32_t n_instructions)
6593 {
6594         uint32_t i, pos = 0;
6595
6596         /* Eliminate the invalid instructions that have been optimized out. */
6597         for (i = 0; i < n_instructions; i++) {
6598                 struct instruction *instr = &instructions[i];
6599                 struct instruction_data *data = &instruction_data[i];
6600
6601                 if (data->invalid)
6602                         continue;
6603
6604                 if (i != pos) {
6605                         memcpy(&instructions[pos], instr, sizeof(*instr));
6606                         memcpy(&instruction_data[pos], data, sizeof(*data));
6607                 }
6608
6609                 pos++;
6610         }
6611
6612         return pos;
6613 }
6614
6615 static int
6616 instr_pattern_extract_many_search(struct instruction *instr,
6617                                   struct instruction_data *data,
6618                                   uint32_t n_instr,
6619                                   uint32_t *n_pattern_instr)
6620 {
6621         uint32_t i;
6622
6623         for (i = 0; i < n_instr; i++) {
6624                 if (data[i].invalid)
6625                         break;
6626
6627                 if (instr[i].type != INSTR_HDR_EXTRACT)
6628                         break;
6629
6630                 if (i == RTE_DIM(instr->io.hdr.header_id))
6631                         break;
6632
6633                 if (i && data[i].n_users)
6634                         break;
6635         }
6636
6637         if (i < 2)
6638                 return 0;
6639
6640         *n_pattern_instr = i;
6641         return 1;
6642 }
6643
6644 static void
6645 instr_pattern_extract_many_replace(struct instruction *instr,
6646                                    struct instruction_data *data,
6647                                    uint32_t n_instr)
6648 {
6649         uint32_t i;
6650
6651         for (i = 1; i < n_instr; i++) {
6652                 instr[0].type++;
6653                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6654                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6655                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6656
6657                 data[i].invalid = 1;
6658         }
6659 }
6660
6661 static uint32_t
6662 instr_pattern_extract_many_optimize(struct instruction *instructions,
6663                                     struct instruction_data *instruction_data,
6664                                     uint32_t n_instructions)
6665 {
6666         uint32_t i;
6667
6668         for (i = 0; i < n_instructions; ) {
6669                 struct instruction *instr = &instructions[i];
6670                 struct instruction_data *data = &instruction_data[i];
6671                 uint32_t n_instr = 0;
6672                 int detected;
6673
6674                 /* Extract many. */
6675                 detected = instr_pattern_extract_many_search(instr,
6676                                                              data,
6677                                                              n_instructions - i,
6678                                                              &n_instr);
6679                 if (detected) {
6680                         instr_pattern_extract_many_replace(instr,
6681                                                            data,
6682                                                            n_instr);
6683                         i += n_instr;
6684                         continue;
6685                 }
6686
6687                 /* No pattern starting at the current instruction. */
6688                 i++;
6689         }
6690
6691         /* Eliminate the invalid instructions that have been optimized out. */
6692         n_instructions = instr_compact(instructions,
6693                                        instruction_data,
6694                                        n_instructions);
6695
6696         return n_instructions;
6697 }
6698
6699 static int
6700 instr_pattern_emit_many_tx_search(struct instruction *instr,
6701                                   struct instruction_data *data,
6702                                   uint32_t n_instr,
6703                                   uint32_t *n_pattern_instr)
6704 {
6705         uint32_t i;
6706
6707         for (i = 0; i < n_instr; i++) {
6708                 if (data[i].invalid)
6709                         break;
6710
6711                 if (instr[i].type != INSTR_HDR_EMIT)
6712                         break;
6713
6714                 if (i == RTE_DIM(instr->io.hdr.header_id))
6715                         break;
6716
6717                 if (i && data[i].n_users)
6718                         break;
6719         }
6720
6721         if (!i)
6722                 return 0;
6723
6724         if (!instruction_is_tx(instr[i].type))
6725                 return 0;
6726
6727         if (data[i].n_users)
6728                 return 0;
6729
6730         i++;
6731
6732         *n_pattern_instr = i;
6733         return 1;
6734 }
6735
6736 static void
6737 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6738                                    struct instruction_data *data,
6739                                    uint32_t n_instr)
6740 {
6741         uint32_t i;
6742
6743         /* Any emit instruction in addition to the first one. */
6744         for (i = 1; i < n_instr - 1; i++) {
6745                 instr[0].type++;
6746                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6747                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6748                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6749
6750                 data[i].invalid = 1;
6751         }
6752
6753         /* The TX instruction is the last one in the pattern. */
6754         instr[0].type++;
6755         instr[0].io.io.offset = instr[i].io.io.offset;
6756         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6757         data[i].invalid = 1;
6758 }
6759
6760 static uint32_t
6761 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6762                                     struct instruction_data *instruction_data,
6763                                     uint32_t n_instructions)
6764 {
6765         uint32_t i;
6766
6767         for (i = 0; i < n_instructions; ) {
6768                 struct instruction *instr = &instructions[i];
6769                 struct instruction_data *data = &instruction_data[i];
6770                 uint32_t n_instr = 0;
6771                 int detected;
6772
6773                 /* Emit many + TX. */
6774                 detected = instr_pattern_emit_many_tx_search(instr,
6775                                                              data,
6776                                                              n_instructions - i,
6777                                                              &n_instr);
6778                 if (detected) {
6779                         instr_pattern_emit_many_tx_replace(instr,
6780                                                            data,
6781                                                            n_instr);
6782                         i += n_instr;
6783                         continue;
6784                 }
6785
6786                 /* No pattern starting at the current instruction. */
6787                 i++;
6788         }
6789
6790         /* Eliminate the invalid instructions that have been optimized out. */
6791         n_instructions = instr_compact(instructions,
6792                                        instruction_data,
6793                                        n_instructions);
6794
6795         return n_instructions;
6796 }
6797
6798 static uint32_t
6799 action_arg_src_mov_count(struct action *a,
6800                          uint32_t arg_id,
6801                          struct instruction *instructions,
6802                          struct instruction_data *instruction_data,
6803                          uint32_t n_instructions);
6804
6805 static int
6806 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6807                                       struct action *a,
6808                                       struct instruction *instr,
6809                                       struct instruction_data *data,
6810                                       uint32_t n_instr,
6811                                       struct instruction *instructions,
6812                                       struct instruction_data *instruction_data,
6813                                       uint32_t n_instructions,
6814                                       uint32_t *n_pattern_instr)
6815 {
6816         struct header *h;
6817         uint32_t src_field_id, i, j;
6818
6819         /* Prerequisites. */
6820         if (!a || !a->st)
6821                 return 0;
6822
6823         /* First instruction: MOV_HM. */
6824         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6825                 return 0;
6826
6827         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6828         if (!h || h->st->var_size)
6829                 return 0;
6830
6831         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6832                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6833                         break;
6834
6835         if (src_field_id == a->st->n_fields)
6836                 return 0;
6837
6838         if (instr[0].mov.dst.offset ||
6839             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6840             instr[0].mov.src.struct_id ||
6841             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6842             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6843                 return 0;
6844
6845         if ((n_instr < h->st->n_fields + 1) ||
6846              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6847                 return 0;
6848
6849         /* Subsequent instructions: MOV_HM. */
6850         for (i = 1; i < h->st->n_fields; i++)
6851                 if (data[i].invalid ||
6852                     data[i].n_users ||
6853                     (instr[i].type != INSTR_MOV_HM) ||
6854                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6855                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6856                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6857                     instr[i].mov.src.struct_id ||
6858                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6859                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6860                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6861                         return 0;
6862
6863         /* Last instruction: HDR_VALIDATE. */
6864         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6865             (instr[i].valid.header_id != h->id))
6866                 return 0;
6867
6868         /* Check that none of the action args that are used as source for this
6869          * DMA transfer are not used as source in any other mov instruction.
6870          */
6871         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6872                 uint32_t n_users;
6873
6874                 n_users = action_arg_src_mov_count(a,
6875                                                    j,
6876                                                    instructions,
6877                                                    instruction_data,
6878                                                    n_instructions);
6879                 if (n_users > 1)
6880                         return 0;
6881         }
6882
6883         *n_pattern_instr = 1 + i;
6884         return 1;
6885 }
6886
6887 static void
6888 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6889                                        struct action *a,
6890                                        struct instruction *instr,
6891                                        struct instruction_data *data,
6892                                        uint32_t n_instr)
6893 {
6894         struct header *h;
6895         uint32_t src_field_id, src_offset, i;
6896
6897         /* Read from the instructions before they are modified. */
6898         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6899         if (!h)
6900                 return;
6901
6902         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6903                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6904                         break;
6905
6906         if (src_field_id == a->st->n_fields)
6907                 return;
6908
6909         src_offset = instr[0].mov.src.offset;
6910
6911         /* Modify the instructions. */
6912         instr[0].type = INSTR_DMA_HT;
6913         instr[0].dma.dst.header_id[0] = h->id;
6914         instr[0].dma.dst.struct_id[0] = h->struct_id;
6915         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6916         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6917
6918         for (i = 1; i < n_instr; i++)
6919                 data[i].invalid = 1;
6920
6921         /* Update the endianness of the action arguments to header endianness. */
6922         for (i = 0; i < h->st->n_fields; i++)
6923                 a->args_endianness[src_field_id + i] = 1;
6924 }
6925
6926 static uint32_t
6927 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6928                                         struct action *a,
6929                                         struct instruction *instructions,
6930                                         struct instruction_data *instruction_data,
6931                                         uint32_t n_instructions)
6932 {
6933         uint32_t i;
6934
6935         if (!a || !a->st)
6936                 return n_instructions;
6937
6938         for (i = 0; i < n_instructions; ) {
6939                 struct instruction *instr = &instructions[i];
6940                 struct instruction_data *data = &instruction_data[i];
6941                 uint32_t n_instr = 0;
6942                 int detected;
6943
6944                 /* Mov all + validate. */
6945                 detected = instr_pattern_mov_all_validate_search(p,
6946                                                                  a,
6947                                                                  instr,
6948                                                                  data,
6949                                                                  n_instructions - i,
6950                                                                  instructions,
6951                                                                  instruction_data,
6952                                                                  n_instructions,
6953                                                                  &n_instr);
6954                 if (detected) {
6955                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6956                         i += n_instr;
6957                         continue;
6958                 }
6959
6960                 /* No pattern starting at the current instruction. */
6961                 i++;
6962         }
6963
6964         /* Eliminate the invalid instructions that have been optimized out. */
6965         n_instructions = instr_compact(instructions,
6966                                        instruction_data,
6967                                        n_instructions);
6968
6969         return n_instructions;
6970 }
6971
6972 static int
6973 instr_pattern_dma_many_search(struct instruction *instr,
6974                               struct instruction_data *data,
6975                               uint32_t n_instr,
6976                               uint32_t *n_pattern_instr)
6977 {
6978         uint32_t i;
6979
6980         for (i = 0; i < n_instr; i++) {
6981                 if (data[i].invalid)
6982                         break;
6983
6984                 if (instr[i].type != INSTR_DMA_HT)
6985                         break;
6986
6987                 if (i == RTE_DIM(instr->dma.dst.header_id))
6988                         break;
6989
6990                 if (i && data[i].n_users)
6991                         break;
6992         }
6993
6994         if (i < 2)
6995                 return 0;
6996
6997         *n_pattern_instr = i;
6998         return 1;
6999 }
7000
7001 static void
7002 instr_pattern_dma_many_replace(struct instruction *instr,
7003                                struct instruction_data *data,
7004                                uint32_t n_instr)
7005 {
7006         uint32_t i;
7007
7008         for (i = 1; i < n_instr; i++) {
7009                 instr[0].type++;
7010                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7011                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7012                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7013                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7014
7015                 data[i].invalid = 1;
7016         }
7017 }
7018
7019 static uint32_t
7020 instr_pattern_dma_many_optimize(struct instruction *instructions,
7021                struct instruction_data *instruction_data,
7022                uint32_t n_instructions)
7023 {
7024         uint32_t i;
7025
7026         for (i = 0; i < n_instructions; ) {
7027                 struct instruction *instr = &instructions[i];
7028                 struct instruction_data *data = &instruction_data[i];
7029                 uint32_t n_instr = 0;
7030                 int detected;
7031
7032                 /* DMA many. */
7033                 detected = instr_pattern_dma_many_search(instr,
7034                                                          data,
7035                                                          n_instructions - i,
7036                                                          &n_instr);
7037                 if (detected) {
7038                         instr_pattern_dma_many_replace(instr, data, n_instr);
7039                         i += n_instr;
7040                         continue;
7041                 }
7042
7043                 /* No pattern starting at the current instruction. */
7044                 i++;
7045         }
7046
7047         /* Eliminate the invalid instructions that have been optimized out. */
7048         n_instructions = instr_compact(instructions,
7049                                        instruction_data,
7050                                        n_instructions);
7051
7052         return n_instructions;
7053 }
7054
7055 static uint32_t
7056 instr_optimize(struct rte_swx_pipeline *p,
7057                struct action *a,
7058                struct instruction *instructions,
7059                struct instruction_data *instruction_data,
7060                uint32_t n_instructions)
7061 {
7062         /* Extract many. */
7063         n_instructions = instr_pattern_extract_many_optimize(instructions,
7064                                                              instruction_data,
7065                                                              n_instructions);
7066
7067         /* Emit many + TX. */
7068         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7069                                                              instruction_data,
7070                                                              n_instructions);
7071
7072         /* Mov all + validate. */
7073         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7074                                                                  a,
7075                                                                  instructions,
7076                                                                  instruction_data,
7077                                                                  n_instructions);
7078
7079         /* DMA many. */
7080         n_instructions = instr_pattern_dma_many_optimize(instructions,
7081                                                          instruction_data,
7082                                                          n_instructions);
7083
7084         return n_instructions;
7085 }
7086
7087 static int
7088 instruction_config(struct rte_swx_pipeline *p,
7089                    struct action *a,
7090                    const char **instructions,
7091                    uint32_t n_instructions)
7092 {
7093         struct instruction *instr = NULL;
7094         struct instruction_data *data = NULL;
7095         int err = 0;
7096         uint32_t i;
7097
7098         CHECK(n_instructions, EINVAL);
7099         CHECK(instructions, EINVAL);
7100         for (i = 0; i < n_instructions; i++)
7101                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7102
7103         /* Memory allocation. */
7104         instr = calloc(n_instructions, sizeof(struct instruction));
7105         if (!instr) {
7106                 err = -ENOMEM;
7107                 goto error;
7108         }
7109
7110         data = calloc(n_instructions, sizeof(struct instruction_data));
7111         if (!data) {
7112                 err = -ENOMEM;
7113                 goto error;
7114         }
7115
7116         for (i = 0; i < n_instructions; i++) {
7117                 char *string = strdup(instructions[i]);
7118                 if (!string) {
7119                         err = -ENOMEM;
7120                         goto error;
7121                 }
7122
7123                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7124                 if (err) {
7125                         free(string);
7126                         goto error;
7127                 }
7128
7129                 free(string);
7130         }
7131
7132         err = instr_label_check(data, n_instructions);
7133         if (err)
7134                 goto error;
7135
7136         err = instr_verify(p, a, instr, data, n_instructions);
7137         if (err)
7138                 goto error;
7139
7140         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7141
7142         err = instr_jmp_resolve(instr, data, n_instructions);
7143         if (err)
7144                 goto error;
7145
7146         if (a) {
7147                 a->instructions = instr;
7148                 a->n_instructions = n_instructions;
7149         } else {
7150                 p->instructions = instr;
7151                 p->n_instructions = n_instructions;
7152         }
7153
7154         free(data);
7155         return 0;
7156
7157 error:
7158         free(data);
7159         free(instr);
7160         return err;
7161 }
7162
7163 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7164
7165 static instr_exec_t instruction_table[] = {
7166         [INSTR_RX] = instr_rx_exec,
7167         [INSTR_TX] = instr_tx_exec,
7168         [INSTR_TX_I] = instr_tx_i_exec,
7169
7170         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7171         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7172         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7173         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7174         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7175         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7176         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7177         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7178         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7179         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7180
7181         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7182         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7183         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7184         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7185         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7186         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7187         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7188         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7189         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7190
7191         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7192         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7193
7194         [INSTR_MOV] = instr_mov_exec,
7195         [INSTR_MOV_MH] = instr_mov_mh_exec,
7196         [INSTR_MOV_HM] = instr_mov_hm_exec,
7197         [INSTR_MOV_HH] = instr_mov_hh_exec,
7198         [INSTR_MOV_I] = instr_mov_i_exec,
7199
7200         [INSTR_DMA_HT] = instr_dma_ht_exec,
7201         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7202         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7203         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7204         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7205         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7206         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7207         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7208
7209         [INSTR_ALU_ADD] = instr_alu_add_exec,
7210         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7211         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7212         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7213         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7214         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7215
7216         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7217         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7218         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7219         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7220         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7221         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7222
7223         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7224         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7225         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7226         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7227
7228         [INSTR_ALU_AND] = instr_alu_and_exec,
7229         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7230         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7231         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7232         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7233
7234         [INSTR_ALU_OR] = instr_alu_or_exec,
7235         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7236         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7237         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7238         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7239
7240         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7241         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7242         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7243         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7244         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7245
7246         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7247         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7248         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7249         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7250         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7251         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7252
7253         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7254         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7255         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7256         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7257         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7258         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7259
7260         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7261         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7262         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7263
7264         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7265         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7266         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7267         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7268         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7269         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7270
7271         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7272         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7273         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7274         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7275         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7276         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7277         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7278         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7279         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7280
7281         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7282         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7283         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7284         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7285         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7286         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7287         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7288         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7289         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7290
7291         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7292         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7293         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7294
7295         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7296         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7297         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7298         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7299         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7300         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7301         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7302         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7303         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7304         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7305         [INSTR_METER_IMM] = instr_meter_imm_exec,
7306         [INSTR_METER_IMI] = instr_meter_imi_exec,
7307
7308         [INSTR_TABLE] = instr_table_exec,
7309         [INSTR_SELECTOR] = instr_selector_exec,
7310         [INSTR_LEARNER] = instr_learner_exec,
7311         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7312         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7313         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7314         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7315
7316         [INSTR_JMP] = instr_jmp_exec,
7317         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7318         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7319         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7320         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7321         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7322         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7323
7324         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7325         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7326         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7327         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7328         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7329
7330         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7331         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7332         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7333         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7334         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7335
7336         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7337         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7338         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7339         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7340         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7341         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7342
7343         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7344         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7345         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7346         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7347         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7348         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7349
7350         [INSTR_RETURN] = instr_return_exec,
7351 };
7352
7353 static inline void
7354 instr_exec(struct rte_swx_pipeline *p)
7355 {
7356         struct thread *t = &p->threads[p->thread_id];
7357         struct instruction *ip = t->ip;
7358         instr_exec_t instr = instruction_table[ip->type];
7359
7360         instr(p);
7361 }
7362
7363 /*
7364  * Action.
7365  */
7366 static struct action *
7367 action_find(struct rte_swx_pipeline *p, const char *name)
7368 {
7369         struct action *elem;
7370
7371         if (!name)
7372                 return NULL;
7373
7374         TAILQ_FOREACH(elem, &p->actions, node)
7375                 if (strcmp(elem->name, name) == 0)
7376                         return elem;
7377
7378         return NULL;
7379 }
7380
7381 static struct action *
7382 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7383 {
7384         struct action *action = NULL;
7385
7386         TAILQ_FOREACH(action, &p->actions, node)
7387                 if (action->id == id)
7388                         return action;
7389
7390         return NULL;
7391 }
7392
7393 static struct field *
7394 action_field_find(struct action *a, const char *name)
7395 {
7396         return a->st ? struct_type_field_find(a->st, name) : NULL;
7397 }
7398
7399 static struct field *
7400 action_field_parse(struct action *action, const char *name)
7401 {
7402         if (name[0] != 't' || name[1] != '.')
7403                 return NULL;
7404
7405         return action_field_find(action, &name[2]);
7406 }
7407
7408 static int
7409 action_has_nbo_args(struct action *a)
7410 {
7411         uint32_t i;
7412
7413         /* Return if the action does not have any args. */
7414         if (!a->st)
7415                 return 0; /* FALSE */
7416
7417         for (i = 0; i < a->st->n_fields; i++)
7418                 if (a->args_endianness[i])
7419                         return 1; /* TRUE */
7420
7421         return 0; /* FALSE */
7422 }
7423
7424 static int
7425 action_does_learning(struct action *a)
7426 {
7427         uint32_t i;
7428
7429         for (i = 0; i < a->n_instructions; i++)
7430                 switch (a->instructions[i].type) {
7431                 case INSTR_LEARNER_LEARN:
7432                         return 1; /* TRUE */
7433
7434                 case INSTR_LEARNER_FORGET:
7435                         return 1; /* TRUE */
7436
7437                 default:
7438                         continue;
7439                 }
7440
7441         return 0; /* FALSE */
7442 }
7443
7444 int
7445 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7446                                const char *name,
7447                                const char *args_struct_type_name,
7448                                const char **instructions,
7449                                uint32_t n_instructions)
7450 {
7451         struct struct_type *args_struct_type = NULL;
7452         struct action *a;
7453         int err;
7454
7455         CHECK(p, EINVAL);
7456
7457         CHECK_NAME(name, EINVAL);
7458         CHECK(!action_find(p, name), EEXIST);
7459
7460         if (args_struct_type_name) {
7461                 CHECK_NAME(args_struct_type_name, EINVAL);
7462                 args_struct_type = struct_type_find(p, args_struct_type_name);
7463                 CHECK(args_struct_type, EINVAL);
7464                 CHECK(!args_struct_type->var_size, EINVAL);
7465         }
7466
7467         /* Node allocation. */
7468         a = calloc(1, sizeof(struct action));
7469         CHECK(a, ENOMEM);
7470         if (args_struct_type) {
7471                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7472                 if (!a->args_endianness) {
7473                         free(a);
7474                         CHECK(0, ENOMEM);
7475                 }
7476         }
7477
7478         /* Node initialization. */
7479         strcpy(a->name, name);
7480         a->st = args_struct_type;
7481         a->id = p->n_actions;
7482
7483         /* Instruction translation. */
7484         err = instruction_config(p, a, instructions, n_instructions);
7485         if (err) {
7486                 free(a->args_endianness);
7487                 free(a);
7488                 return err;
7489         }
7490
7491         /* Node add to tailq. */
7492         TAILQ_INSERT_TAIL(&p->actions, a, node);
7493         p->n_actions++;
7494
7495         return 0;
7496 }
7497
7498 static int
7499 action_build(struct rte_swx_pipeline *p)
7500 {
7501         struct action *action;
7502
7503         p->action_instructions = calloc(p->n_actions,
7504                                         sizeof(struct instruction *));
7505         CHECK(p->action_instructions, ENOMEM);
7506
7507         TAILQ_FOREACH(action, &p->actions, node)
7508                 p->action_instructions[action->id] = action->instructions;
7509
7510         return 0;
7511 }
7512
7513 static void
7514 action_build_free(struct rte_swx_pipeline *p)
7515 {
7516         free(p->action_instructions);
7517         p->action_instructions = NULL;
7518 }
7519
7520 static void
7521 action_free(struct rte_swx_pipeline *p)
7522 {
7523         action_build_free(p);
7524
7525         for ( ; ; ) {
7526                 struct action *action;
7527
7528                 action = TAILQ_FIRST(&p->actions);
7529                 if (!action)
7530                         break;
7531
7532                 TAILQ_REMOVE(&p->actions, action, node);
7533                 free(action->instructions);
7534                 free(action);
7535         }
7536 }
7537
7538 static uint32_t
7539 action_arg_src_mov_count(struct action *a,
7540                          uint32_t arg_id,
7541                          struct instruction *instructions,
7542                          struct instruction_data *instruction_data,
7543                          uint32_t n_instructions)
7544 {
7545         uint32_t offset, n_users = 0, i;
7546
7547         if (!a->st ||
7548             (arg_id >= a->st->n_fields) ||
7549             !instructions ||
7550             !instruction_data ||
7551             !n_instructions)
7552                 return 0;
7553
7554         offset = a->st->fields[arg_id].offset / 8;
7555
7556         for (i = 0; i < n_instructions; i++) {
7557                 struct instruction *instr = &instructions[i];
7558                 struct instruction_data *data = &instruction_data[i];
7559
7560                 if (data->invalid ||
7561                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7562                     instr->mov.src.struct_id ||
7563                     (instr->mov.src.offset != offset))
7564                         continue;
7565
7566                 n_users++;
7567         }
7568
7569         return n_users;
7570 }
7571
7572 /*
7573  * Table.
7574  */
7575 static struct table_type *
7576 table_type_find(struct rte_swx_pipeline *p, const char *name)
7577 {
7578         struct table_type *elem;
7579
7580         TAILQ_FOREACH(elem, &p->table_types, node)
7581                 if (strcmp(elem->name, name) == 0)
7582                         return elem;
7583
7584         return NULL;
7585 }
7586
7587 static struct table_type *
7588 table_type_resolve(struct rte_swx_pipeline *p,
7589                    const char *recommended_type_name,
7590                    enum rte_swx_table_match_type match_type)
7591 {
7592         struct table_type *elem;
7593
7594         /* Only consider the recommended type if the match type is correct. */
7595         if (recommended_type_name)
7596                 TAILQ_FOREACH(elem, &p->table_types, node)
7597                         if (!strcmp(elem->name, recommended_type_name) &&
7598                             (elem->match_type == match_type))
7599                                 return elem;
7600
7601         /* Ignore the recommended type and get the first element with this match
7602          * type.
7603          */
7604         TAILQ_FOREACH(elem, &p->table_types, node)
7605                 if (elem->match_type == match_type)
7606                         return elem;
7607
7608         return NULL;
7609 }
7610
7611 static struct table *
7612 table_find(struct rte_swx_pipeline *p, const char *name)
7613 {
7614         struct table *elem;
7615
7616         TAILQ_FOREACH(elem, &p->tables, node)
7617                 if (strcmp(elem->name, name) == 0)
7618                         return elem;
7619
7620         return NULL;
7621 }
7622
7623 static struct table *
7624 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7625 {
7626         struct table *table = NULL;
7627
7628         TAILQ_FOREACH(table, &p->tables, node)
7629                 if (table->id == id)
7630                         return table;
7631
7632         return NULL;
7633 }
7634
7635 int
7636 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7637                                      const char *name,
7638                                      enum rte_swx_table_match_type match_type,
7639                                      struct rte_swx_table_ops *ops)
7640 {
7641         struct table_type *elem;
7642
7643         CHECK(p, EINVAL);
7644
7645         CHECK_NAME(name, EINVAL);
7646         CHECK(!table_type_find(p, name), EEXIST);
7647
7648         CHECK(ops, EINVAL);
7649         CHECK(ops->create, EINVAL);
7650         CHECK(ops->lkp, EINVAL);
7651         CHECK(ops->free, EINVAL);
7652
7653         /* Node allocation. */
7654         elem = calloc(1, sizeof(struct table_type));
7655         CHECK(elem, ENOMEM);
7656
7657         /* Node initialization. */
7658         strcpy(elem->name, name);
7659         elem->match_type = match_type;
7660         memcpy(&elem->ops, ops, sizeof(*ops));
7661
7662         /* Node add to tailq. */
7663         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7664
7665         return 0;
7666 }
7667
7668 static int
7669 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7670                          uint32_t n_fields,
7671                          enum rte_swx_table_match_type *match_type)
7672 {
7673         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7674
7675         for (i = 0; i < n_fields; i++) {
7676                 struct rte_swx_match_field_params  *f = &fields[i];
7677
7678                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7679                         n_fields_em++;
7680
7681                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7682                         n_fields_lpm++;
7683         }
7684
7685         if ((n_fields_lpm > 1) ||
7686             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7687                 return -EINVAL;
7688
7689         *match_type = (n_fields_em == n_fields) ?
7690                        RTE_SWX_TABLE_MATCH_EXACT :
7691                        RTE_SWX_TABLE_MATCH_WILDCARD;
7692
7693         return 0;
7694 }
7695
7696 static int
7697 table_match_fields_check(struct rte_swx_pipeline *p,
7698                          struct rte_swx_pipeline_table_params *params,
7699                          struct header **header)
7700 {
7701         struct header *h0 = NULL;
7702         struct field *hf, *mf;
7703         uint32_t *offset = NULL, i;
7704         int status = 0;
7705
7706         /* Return if no match fields. */
7707         if (!params->n_fields) {
7708                 if (params->fields) {
7709                         status = -EINVAL;
7710                         goto end;
7711                 }
7712
7713                 if (header)
7714                         *header = NULL;
7715
7716                 return 0;
7717         }
7718
7719         /* Memory allocation. */
7720         offset = calloc(params->n_fields, sizeof(uint32_t));
7721         if (!offset) {
7722                 status = -ENOMEM;
7723                 goto end;
7724         }
7725
7726         /* Check that all the match fields belong to either the same header or
7727          * to the meta-data.
7728          */
7729         hf = header_field_parse(p, params->fields[0].name, &h0);
7730         mf = metadata_field_parse(p, params->fields[0].name);
7731         if ((!hf && !mf) || (hf && hf->var_size)) {
7732                 status = -EINVAL;
7733                 goto end;
7734         }
7735
7736         offset[0] = h0 ? hf->offset : mf->offset;
7737
7738         for (i = 1; i < params->n_fields; i++)
7739                 if (h0) {
7740                         struct header *h;
7741
7742                         hf = header_field_parse(p, params->fields[i].name, &h);
7743                         if (!hf || (h->id != h0->id) || hf->var_size) {
7744                                 status = -EINVAL;
7745                                 goto end;
7746                         }
7747
7748                         offset[i] = hf->offset;
7749                 } else {
7750                         mf = metadata_field_parse(p, params->fields[i].name);
7751                         if (!mf) {
7752                                 status = -EINVAL;
7753                                 goto end;
7754                         }
7755
7756                         offset[i] = mf->offset;
7757                 }
7758
7759         /* Check that there are no duplicated match fields. */
7760         for (i = 0; i < params->n_fields; i++) {
7761                 uint32_t j;
7762
7763                 for (j = 0; j < i; j++)
7764                         if (offset[j] == offset[i]) {
7765                                 status = -EINVAL;
7766                                 goto end;
7767                         }
7768         }
7769
7770         /* Return. */
7771         if (header)
7772                 *header = h0;
7773
7774 end:
7775         free(offset);
7776         return status;
7777 }
7778
7779 int
7780 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7781                               const char *name,
7782                               struct rte_swx_pipeline_table_params *params,
7783                               const char *recommended_table_type_name,
7784                               const char *args,
7785                               uint32_t size)
7786 {
7787         struct table_type *type;
7788         struct table *t;
7789         struct action *default_action;
7790         struct header *header = NULL;
7791         uint32_t action_data_size_max = 0, i;
7792         int status = 0;
7793
7794         CHECK(p, EINVAL);
7795
7796         CHECK_NAME(name, EINVAL);
7797         CHECK(!table_find(p, name), EEXIST);
7798         CHECK(!selector_find(p, name), EEXIST);
7799         CHECK(!learner_find(p, name), EEXIST);
7800
7801         CHECK(params, EINVAL);
7802
7803         /* Match checks. */
7804         status = table_match_fields_check(p, params, &header);
7805         if (status)
7806                 return status;
7807
7808         /* Action checks. */
7809         CHECK(params->n_actions, EINVAL);
7810         CHECK(params->action_names, EINVAL);
7811         for (i = 0; i < params->n_actions; i++) {
7812                 const char *action_name = params->action_names[i];
7813                 struct action *a;
7814                 uint32_t action_data_size;
7815
7816                 CHECK_NAME(action_name, EINVAL);
7817
7818                 a = action_find(p, action_name);
7819                 CHECK(a, EINVAL);
7820                 CHECK(!action_does_learning(a), EINVAL);
7821
7822                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7823                 if (action_data_size > action_data_size_max)
7824                         action_data_size_max = action_data_size;
7825         }
7826
7827         CHECK_NAME(params->default_action_name, EINVAL);
7828         for (i = 0; i < p->n_actions; i++)
7829                 if (!strcmp(params->action_names[i],
7830                             params->default_action_name))
7831                         break;
7832         CHECK(i < params->n_actions, EINVAL);
7833         default_action = action_find(p, params->default_action_name);
7834         CHECK((default_action->st && params->default_action_data) ||
7835               !params->default_action_data, EINVAL);
7836
7837         /* Table type checks. */
7838         if (recommended_table_type_name)
7839                 CHECK_NAME(recommended_table_type_name, EINVAL);
7840
7841         if (params->n_fields) {
7842                 enum rte_swx_table_match_type match_type;
7843
7844                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7845                 if (status)
7846                         return status;
7847
7848                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7849                 CHECK(type, EINVAL);
7850         } else {
7851                 type = NULL;
7852         }
7853
7854         /* Memory allocation. */
7855         t = calloc(1, sizeof(struct table));
7856         CHECK(t, ENOMEM);
7857
7858         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7859         if (!t->fields) {
7860                 free(t);
7861                 CHECK(0, ENOMEM);
7862         }
7863
7864         t->actions = calloc(params->n_actions, sizeof(struct action *));
7865         if (!t->actions) {
7866                 free(t->fields);
7867                 free(t);
7868                 CHECK(0, ENOMEM);
7869         }
7870
7871         if (action_data_size_max) {
7872                 t->default_action_data = calloc(1, action_data_size_max);
7873                 if (!t->default_action_data) {
7874                         free(t->actions);
7875                         free(t->fields);
7876                         free(t);
7877                         CHECK(0, ENOMEM);
7878                 }
7879         }
7880
7881         /* Node initialization. */
7882         strcpy(t->name, name);
7883         if (args && args[0])
7884                 strcpy(t->args, args);
7885         t->type = type;
7886
7887         for (i = 0; i < params->n_fields; i++) {
7888                 struct rte_swx_match_field_params *field = &params->fields[i];
7889                 struct match_field *f = &t->fields[i];
7890
7891                 f->match_type = field->match_type;
7892                 f->field = header ?
7893                         header_field_parse(p, field->name, NULL) :
7894                         metadata_field_parse(p, field->name);
7895         }
7896         t->n_fields = params->n_fields;
7897         t->header = header;
7898
7899         for (i = 0; i < params->n_actions; i++)
7900                 t->actions[i] = action_find(p, params->action_names[i]);
7901         t->default_action = default_action;
7902         if (default_action->st)
7903                 memcpy(t->default_action_data,
7904                        params->default_action_data,
7905                        default_action->st->n_bits / 8);
7906         t->n_actions = params->n_actions;
7907         t->default_action_is_const = params->default_action_is_const;
7908         t->action_data_size_max = action_data_size_max;
7909
7910         t->size = size;
7911         t->id = p->n_tables;
7912
7913         /* Node add to tailq. */
7914         TAILQ_INSERT_TAIL(&p->tables, t, node);
7915         p->n_tables++;
7916
7917         return 0;
7918 }
7919
7920 static struct rte_swx_table_params *
7921 table_params_get(struct table *table)
7922 {
7923         struct rte_swx_table_params *params;
7924         struct field *first, *last;
7925         uint8_t *key_mask;
7926         uint32_t key_size, key_offset, action_data_size, i;
7927
7928         /* Memory allocation. */
7929         params = calloc(1, sizeof(struct rte_swx_table_params));
7930         if (!params)
7931                 return NULL;
7932
7933         /* Find first (smallest offset) and last (biggest offset) match fields. */
7934         first = table->fields[0].field;
7935         last = table->fields[0].field;
7936
7937         for (i = 0; i < table->n_fields; i++) {
7938                 struct field *f = table->fields[i].field;
7939
7940                 if (f->offset < first->offset)
7941                         first = f;
7942
7943                 if (f->offset > last->offset)
7944                         last = f;
7945         }
7946
7947         /* Key offset and size. */
7948         key_offset = first->offset / 8;
7949         key_size = (last->offset + last->n_bits - first->offset) / 8;
7950
7951         /* Memory allocation. */
7952         key_mask = calloc(1, key_size);
7953         if (!key_mask) {
7954                 free(params);
7955                 return NULL;
7956         }
7957
7958         /* Key mask. */
7959         for (i = 0; i < table->n_fields; i++) {
7960                 struct field *f = table->fields[i].field;
7961                 uint32_t start = (f->offset - first->offset) / 8;
7962                 size_t size = f->n_bits / 8;
7963
7964                 memset(&key_mask[start], 0xFF, size);
7965         }
7966
7967         /* Action data size. */
7968         action_data_size = 0;
7969         for (i = 0; i < table->n_actions; i++) {
7970                 struct action *action = table->actions[i];
7971                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7972
7973                 if (ads > action_data_size)
7974                         action_data_size = ads;
7975         }
7976
7977         /* Fill in. */
7978         params->match_type = table->type->match_type;
7979         params->key_size = key_size;
7980         params->key_offset = key_offset;
7981         params->key_mask0 = key_mask;
7982         params->action_data_size = action_data_size;
7983         params->n_keys_max = table->size;
7984
7985         return params;
7986 }
7987
7988 static void
7989 table_params_free(struct rte_swx_table_params *params)
7990 {
7991         if (!params)
7992                 return;
7993
7994         free(params->key_mask0);
7995         free(params);
7996 }
7997
7998 static int
7999 table_stub_lkp(void *table __rte_unused,
8000                void *mailbox __rte_unused,
8001                uint8_t **key __rte_unused,
8002                uint64_t *action_id __rte_unused,
8003                uint8_t **action_data __rte_unused,
8004                int *hit)
8005 {
8006         *hit = 0;
8007         return 1; /* DONE. */
8008 }
8009
8010 static int
8011 table_build(struct rte_swx_pipeline *p)
8012 {
8013         uint32_t i;
8014
8015         /* Per pipeline: table statistics. */
8016         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8017         CHECK(p->table_stats, ENOMEM);
8018
8019         for (i = 0; i < p->n_tables; i++) {
8020                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8021                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8022         }
8023
8024         /* Per thread: table runt-time. */
8025         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8026                 struct thread *t = &p->threads[i];
8027                 struct table *table;
8028
8029                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8030                 CHECK(t->tables, ENOMEM);
8031
8032                 TAILQ_FOREACH(table, &p->tables, node) {
8033                         struct table_runtime *r = &t->tables[table->id];
8034
8035                         if (table->type) {
8036                                 uint64_t size;
8037
8038                                 size = table->type->ops.mailbox_size_get();
8039
8040                                 /* r->func. */
8041                                 r->func = table->type->ops.lkp;
8042
8043                                 /* r->mailbox. */
8044                                 if (size) {
8045                                         r->mailbox = calloc(1, size);
8046                                         CHECK(r->mailbox, ENOMEM);
8047                                 }
8048
8049                                 /* r->key. */
8050                                 r->key = table->header ?
8051                                         &t->structs[table->header->struct_id] :
8052                                         &t->structs[p->metadata_struct_id];
8053                         } else {
8054                                 r->func = table_stub_lkp;
8055                         }
8056                 }
8057         }
8058
8059         return 0;
8060 }
8061
8062 static void
8063 table_build_free(struct rte_swx_pipeline *p)
8064 {
8065         uint32_t i;
8066
8067         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8068                 struct thread *t = &p->threads[i];
8069                 uint32_t j;
8070
8071                 if (!t->tables)
8072                         continue;
8073
8074                 for (j = 0; j < p->n_tables; j++) {
8075                         struct table_runtime *r = &t->tables[j];
8076
8077                         free(r->mailbox);
8078                 }
8079
8080                 free(t->tables);
8081                 t->tables = NULL;
8082         }
8083
8084         if (p->table_stats) {
8085                 for (i = 0; i < p->n_tables; i++)
8086                         free(p->table_stats[i].n_pkts_action);
8087
8088                 free(p->table_stats);
8089         }
8090 }
8091
8092 static void
8093 table_free(struct rte_swx_pipeline *p)
8094 {
8095         table_build_free(p);
8096
8097         /* Tables. */
8098         for ( ; ; ) {
8099                 struct table *elem;
8100
8101                 elem = TAILQ_FIRST(&p->tables);
8102                 if (!elem)
8103                         break;
8104
8105                 TAILQ_REMOVE(&p->tables, elem, node);
8106                 free(elem->fields);
8107                 free(elem->actions);
8108                 free(elem->default_action_data);
8109                 free(elem);
8110         }
8111
8112         /* Table types. */
8113         for ( ; ; ) {
8114                 struct table_type *elem;
8115
8116                 elem = TAILQ_FIRST(&p->table_types);
8117                 if (!elem)
8118                         break;
8119
8120                 TAILQ_REMOVE(&p->table_types, elem, node);
8121                 free(elem);
8122         }
8123 }
8124
8125 /*
8126  * Selector.
8127  */
8128 static struct selector *
8129 selector_find(struct rte_swx_pipeline *p, const char *name)
8130 {
8131         struct selector *s;
8132
8133         TAILQ_FOREACH(s, &p->selectors, node)
8134                 if (strcmp(s->name, name) == 0)
8135                         return s;
8136
8137         return NULL;
8138 }
8139
8140 static struct selector *
8141 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8142 {
8143         struct selector *s = NULL;
8144
8145         TAILQ_FOREACH(s, &p->selectors, node)
8146                 if (s->id == id)
8147                         return s;
8148
8149         return NULL;
8150 }
8151
8152 static int
8153 selector_fields_check(struct rte_swx_pipeline *p,
8154                       struct rte_swx_pipeline_selector_params *params,
8155                       struct header **header)
8156 {
8157         struct header *h0 = NULL;
8158         struct field *hf, *mf;
8159         uint32_t i;
8160
8161         /* Return if no selector fields. */
8162         if (!params->n_selector_fields || !params->selector_field_names)
8163                 return -EINVAL;
8164
8165         /* Check that all the selector fields either belong to the same header
8166          * or are all meta-data fields.
8167          */
8168         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8169         mf = metadata_field_parse(p, params->selector_field_names[0]);
8170         if (!hf && !mf)
8171                 return -EINVAL;
8172
8173         for (i = 1; i < params->n_selector_fields; i++)
8174                 if (h0) {
8175                         struct header *h;
8176
8177                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8178                         if (!hf || (h->id != h0->id))
8179                                 return -EINVAL;
8180                 } else {
8181                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8182                         if (!mf)
8183                                 return -EINVAL;
8184                 }
8185
8186         /* Check that there are no duplicated match fields. */
8187         for (i = 0; i < params->n_selector_fields; i++) {
8188                 const char *field_name = params->selector_field_names[i];
8189                 uint32_t j;
8190
8191                 for (j = i + 1; j < params->n_selector_fields; j++)
8192                         if (!strcmp(params->selector_field_names[j], field_name))
8193                                 return -EINVAL;
8194         }
8195
8196         /* Return. */
8197         if (header)
8198                 *header = h0;
8199
8200         return 0;
8201 }
8202
8203 int
8204 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8205                                  const char *name,
8206                                  struct rte_swx_pipeline_selector_params *params)
8207 {
8208         struct selector *s;
8209         struct header *selector_header = NULL;
8210         struct field *group_id_field, *member_id_field;
8211         uint32_t i;
8212         int status = 0;
8213
8214         CHECK(p, EINVAL);
8215
8216         CHECK_NAME(name, EINVAL);
8217         CHECK(!table_find(p, name), EEXIST);
8218         CHECK(!selector_find(p, name), EEXIST);
8219         CHECK(!learner_find(p, name), EEXIST);
8220
8221         CHECK(params, EINVAL);
8222
8223         CHECK_NAME(params->group_id_field_name, EINVAL);
8224         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8225         CHECK(group_id_field, EINVAL);
8226
8227         for (i = 0; i < params->n_selector_fields; i++) {
8228                 const char *field_name = params->selector_field_names[i];
8229
8230                 CHECK_NAME(field_name, EINVAL);
8231         }
8232         status = selector_fields_check(p, params, &selector_header);
8233         if (status)
8234                 return status;
8235
8236         CHECK_NAME(params->member_id_field_name, EINVAL);
8237         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8238         CHECK(member_id_field, EINVAL);
8239
8240         CHECK(params->n_groups_max, EINVAL);
8241
8242         CHECK(params->n_members_per_group_max, EINVAL);
8243
8244         /* Memory allocation. */
8245         s = calloc(1, sizeof(struct selector));
8246         if (!s) {
8247                 status = -ENOMEM;
8248                 goto error;
8249         }
8250
8251         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8252         if (!s->selector_fields) {
8253                 status = -ENOMEM;
8254                 goto error;
8255         }
8256
8257         /* Node initialization. */
8258         strcpy(s->name, name);
8259
8260         s->group_id_field = group_id_field;
8261
8262         for (i = 0; i < params->n_selector_fields; i++) {
8263                 const char *field_name = params->selector_field_names[i];
8264
8265                 s->selector_fields[i] = selector_header ?
8266                         header_field_parse(p, field_name, NULL) :
8267                         metadata_field_parse(p, field_name);
8268         }
8269
8270         s->n_selector_fields = params->n_selector_fields;
8271
8272         s->selector_header = selector_header;
8273
8274         s->member_id_field = member_id_field;
8275
8276         s->n_groups_max = params->n_groups_max;
8277
8278         s->n_members_per_group_max = params->n_members_per_group_max;
8279
8280         s->id = p->n_selectors;
8281
8282         /* Node add to tailq. */
8283         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8284         p->n_selectors++;
8285
8286         return 0;
8287
8288 error:
8289         if (!s)
8290                 return status;
8291
8292         free(s->selector_fields);
8293
8294         free(s);
8295
8296         return status;
8297 }
8298
8299 static void
8300 selector_params_free(struct rte_swx_table_selector_params *params)
8301 {
8302         if (!params)
8303                 return;
8304
8305         free(params->selector_mask);
8306
8307         free(params);
8308 }
8309
8310 static struct rte_swx_table_selector_params *
8311 selector_table_params_get(struct selector *s)
8312 {
8313         struct rte_swx_table_selector_params *params = NULL;
8314         struct field *first, *last;
8315         uint32_t i;
8316
8317         /* Memory allocation. */
8318         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8319         if (!params)
8320                 goto error;
8321
8322         /* Group ID. */
8323         params->group_id_offset = s->group_id_field->offset / 8;
8324
8325         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8326         first = s->selector_fields[0];
8327         last = s->selector_fields[0];
8328
8329         for (i = 0; i < s->n_selector_fields; i++) {
8330                 struct field *f = s->selector_fields[i];
8331
8332                 if (f->offset < first->offset)
8333                         first = f;
8334
8335                 if (f->offset > last->offset)
8336                         last = f;
8337         }
8338
8339         /* Selector offset and size. */
8340         params->selector_offset = first->offset / 8;
8341         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8342
8343         /* Memory allocation. */
8344         params->selector_mask = calloc(1, params->selector_size);
8345         if (!params->selector_mask)
8346                 goto error;
8347
8348         /* Selector mask. */
8349         for (i = 0; i < s->n_selector_fields; i++) {
8350                 struct field *f = s->selector_fields[i];
8351                 uint32_t start = (f->offset - first->offset) / 8;
8352                 size_t size = f->n_bits / 8;
8353
8354                 memset(&params->selector_mask[start], 0xFF, size);
8355         }
8356
8357         /* Member ID. */
8358         params->member_id_offset = s->member_id_field->offset / 8;
8359
8360         /* Maximum number of groups. */
8361         params->n_groups_max = s->n_groups_max;
8362
8363         /* Maximum number of members per group. */
8364         params->n_members_per_group_max = s->n_members_per_group_max;
8365
8366         return params;
8367
8368 error:
8369         selector_params_free(params);
8370         return NULL;
8371 }
8372
8373 static void
8374 selector_build_free(struct rte_swx_pipeline *p)
8375 {
8376         uint32_t i;
8377
8378         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8379                 struct thread *t = &p->threads[i];
8380                 uint32_t j;
8381
8382                 if (!t->selectors)
8383                         continue;
8384
8385                 for (j = 0; j < p->n_selectors; j++) {
8386                         struct selector_runtime *r = &t->selectors[j];
8387
8388                         free(r->mailbox);
8389                 }
8390
8391                 free(t->selectors);
8392                 t->selectors = NULL;
8393         }
8394
8395         free(p->selector_stats);
8396         p->selector_stats = NULL;
8397 }
8398
8399 static int
8400 selector_build(struct rte_swx_pipeline *p)
8401 {
8402         uint32_t i;
8403         int status = 0;
8404
8405         /* Per pipeline: selector statistics. */
8406         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8407         if (!p->selector_stats) {
8408                 status = -ENOMEM;
8409                 goto error;
8410         }
8411
8412         /* Per thread: selector run-time. */
8413         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8414                 struct thread *t = &p->threads[i];
8415                 struct selector *s;
8416
8417                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8418                 if (!t->selectors) {
8419                         status = -ENOMEM;
8420                         goto error;
8421                 }
8422
8423                 TAILQ_FOREACH(s, &p->selectors, node) {
8424                         struct selector_runtime *r = &t->selectors[s->id];
8425                         uint64_t size;
8426
8427                         /* r->mailbox. */
8428                         size = rte_swx_table_selector_mailbox_size_get();
8429                         if (size) {
8430                                 r->mailbox = calloc(1, size);
8431                                 if (!r->mailbox) {
8432                                         status = -ENOMEM;
8433                                         goto error;
8434                                 }
8435                         }
8436
8437                         /* r->group_id_buffer. */
8438                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8439
8440                         /* r->selector_buffer. */
8441                         r->selector_buffer = s->selector_header ?
8442                                 &t->structs[s->selector_header->struct_id] :
8443                                 &t->structs[p->metadata_struct_id];
8444
8445                         /* r->member_id_buffer. */
8446                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8447                 }
8448         }
8449
8450         return 0;
8451
8452 error:
8453         selector_build_free(p);
8454         return status;
8455 }
8456
8457 static void
8458 selector_free(struct rte_swx_pipeline *p)
8459 {
8460         selector_build_free(p);
8461
8462         /* Selector tables. */
8463         for ( ; ; ) {
8464                 struct selector *elem;
8465
8466                 elem = TAILQ_FIRST(&p->selectors);
8467                 if (!elem)
8468                         break;
8469
8470                 TAILQ_REMOVE(&p->selectors, elem, node);
8471                 free(elem->selector_fields);
8472                 free(elem);
8473         }
8474 }
8475
8476 /*
8477  * Learner table.
8478  */
8479 static struct learner *
8480 learner_find(struct rte_swx_pipeline *p, const char *name)
8481 {
8482         struct learner *l;
8483
8484         TAILQ_FOREACH(l, &p->learners, node)
8485                 if (!strcmp(l->name, name))
8486                         return l;
8487
8488         return NULL;
8489 }
8490
8491 static struct learner *
8492 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8493 {
8494         struct learner *l = NULL;
8495
8496         TAILQ_FOREACH(l, &p->learners, node)
8497                 if (l->id == id)
8498                         return l;
8499
8500         return NULL;
8501 }
8502
8503 static int
8504 learner_match_fields_check(struct rte_swx_pipeline *p,
8505                            struct rte_swx_pipeline_learner_params *params,
8506                            struct header **header)
8507 {
8508         struct header *h0 = NULL;
8509         struct field *hf, *mf;
8510         uint32_t i;
8511
8512         /* Return if no match fields. */
8513         if (!params->n_fields || !params->field_names)
8514                 return -EINVAL;
8515
8516         /* Check that all the match fields either belong to the same header
8517          * or are all meta-data fields.
8518          */
8519         hf = header_field_parse(p, params->field_names[0], &h0);
8520         mf = metadata_field_parse(p, params->field_names[0]);
8521         if (!hf && !mf)
8522                 return -EINVAL;
8523
8524         for (i = 1; i < params->n_fields; i++)
8525                 if (h0) {
8526                         struct header *h;
8527
8528                         hf = header_field_parse(p, params->field_names[i], &h);
8529                         if (!hf || (h->id != h0->id))
8530                                 return -EINVAL;
8531                 } else {
8532                         mf = metadata_field_parse(p, params->field_names[i]);
8533                         if (!mf)
8534                                 return -EINVAL;
8535                 }
8536
8537         /* Check that there are no duplicated match fields. */
8538         for (i = 0; i < params->n_fields; i++) {
8539                 const char *field_name = params->field_names[i];
8540                 uint32_t j;
8541
8542                 for (j = i + 1; j < params->n_fields; j++)
8543                         if (!strcmp(params->field_names[j], field_name))
8544                                 return -EINVAL;
8545         }
8546
8547         /* Return. */
8548         if (header)
8549                 *header = h0;
8550
8551         return 0;
8552 }
8553
8554 static int
8555 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8556 {
8557         struct struct_type *mst = p->metadata_st, *ast = a->st;
8558         struct field *mf, *af;
8559         uint32_t mf_pos, i;
8560
8561         if (!ast) {
8562                 if (mf_name)
8563                         return -EINVAL;
8564
8565                 return 0;
8566         }
8567
8568         /* Check that mf_name is the name of a valid meta-data field. */
8569         CHECK_NAME(mf_name, EINVAL);
8570         mf = metadata_field_parse(p, mf_name);
8571         CHECK(mf, EINVAL);
8572
8573         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8574          * all the action arguments.
8575          */
8576         mf_pos = mf - mst->fields;
8577         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8578
8579         /* Check that the size of each of the identified meta-data fields matches exactly the size
8580          * of the corresponding action argument.
8581          */
8582         for (i = 0; i < ast->n_fields; i++) {
8583                 mf = &mst->fields[mf_pos + i];
8584                 af = &ast->fields[i];
8585
8586                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8587         }
8588
8589         return 0;
8590 }
8591
8592 static int
8593 learner_action_learning_check(struct rte_swx_pipeline *p,
8594                               struct action *action,
8595                               const char **action_names,
8596                               uint32_t n_actions)
8597 {
8598         uint32_t i;
8599
8600         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8601          * the action passed as argument to the "learn" instruction) is also enabled for the
8602          * current learner table.
8603          */
8604         for (i = 0; i < action->n_instructions; i++) {
8605                 struct instruction *instr = &action->instructions[i];
8606                 uint32_t found = 0, j;
8607
8608                 if (instr->type != INSTR_LEARNER_LEARN)
8609                         continue;
8610
8611                 for (j = 0; j < n_actions; j++) {
8612                         struct action *a;
8613
8614                         a = action_find(p, action_names[j]);
8615                         if (!a)
8616                                 return -EINVAL;
8617
8618                         if (a->id == instr->learn.action_id)
8619                                 found = 1;
8620                 }
8621
8622                 if (!found)
8623                         return -EINVAL;
8624         }
8625
8626         return 0;
8627 }
8628
8629 int
8630 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8631                               const char *name,
8632                               struct rte_swx_pipeline_learner_params *params,
8633                               uint32_t size,
8634                               uint32_t timeout)
8635 {
8636         struct learner *l = NULL;
8637         struct action *default_action;
8638         struct header *header = NULL;
8639         uint32_t action_data_size_max = 0, i;
8640         int status = 0;
8641
8642         CHECK(p, EINVAL);
8643
8644         CHECK_NAME(name, EINVAL);
8645         CHECK(!table_find(p, name), EEXIST);
8646         CHECK(!selector_find(p, name), EEXIST);
8647         CHECK(!learner_find(p, name), EEXIST);
8648
8649         CHECK(params, EINVAL);
8650
8651         /* Match checks. */
8652         status = learner_match_fields_check(p, params, &header);
8653         if (status)
8654                 return status;
8655
8656         /* Action checks. */
8657         CHECK(params->n_actions, EINVAL);
8658
8659         CHECK(params->action_names, EINVAL);
8660         for (i = 0; i < params->n_actions; i++) {
8661                 const char *action_name = params->action_names[i];
8662                 const char *action_field_name = params->action_field_names[i];
8663                 struct action *a;
8664                 uint32_t action_data_size;
8665
8666                 CHECK_NAME(action_name, EINVAL);
8667
8668                 a = action_find(p, action_name);
8669                 CHECK(a, EINVAL);
8670
8671                 status = learner_action_args_check(p, a, action_field_name);
8672                 if (status)
8673                         return status;
8674
8675                 status = learner_action_learning_check(p,
8676                                                        a,
8677                                                        params->action_names,
8678                                                        params->n_actions);
8679                 if (status)
8680                         return status;
8681
8682                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8683                 if (action_data_size > action_data_size_max)
8684                         action_data_size_max = action_data_size;
8685         }
8686
8687         CHECK_NAME(params->default_action_name, EINVAL);
8688         for (i = 0; i < p->n_actions; i++)
8689                 if (!strcmp(params->action_names[i],
8690                             params->default_action_name))
8691                         break;
8692         CHECK(i < params->n_actions, EINVAL);
8693
8694         default_action = action_find(p, params->default_action_name);
8695         CHECK((default_action->st && params->default_action_data) ||
8696               !params->default_action_data, EINVAL);
8697
8698         /* Any other checks. */
8699         CHECK(size, EINVAL);
8700         CHECK(timeout, EINVAL);
8701
8702         /* Memory allocation. */
8703         l = calloc(1, sizeof(struct learner));
8704         if (!l)
8705                 goto nomem;
8706
8707         l->fields = calloc(params->n_fields, sizeof(struct field *));
8708         if (!l->fields)
8709                 goto nomem;
8710
8711         l->actions = calloc(params->n_actions, sizeof(struct action *));
8712         if (!l->actions)
8713                 goto nomem;
8714
8715         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8716         if (!l->action_arg)
8717                 goto nomem;
8718
8719         if (action_data_size_max) {
8720                 l->default_action_data = calloc(1, action_data_size_max);
8721                 if (!l->default_action_data)
8722                         goto nomem;
8723         }
8724
8725         /* Node initialization. */
8726         strcpy(l->name, name);
8727
8728         for (i = 0; i < params->n_fields; i++) {
8729                 const char *field_name = params->field_names[i];
8730
8731                 l->fields[i] = header ?
8732                         header_field_parse(p, field_name, NULL) :
8733                         metadata_field_parse(p, field_name);
8734         }
8735
8736         l->n_fields = params->n_fields;
8737
8738         l->header = header;
8739
8740         for (i = 0; i < params->n_actions; i++) {
8741                 const char *mf_name = params->action_field_names[i];
8742
8743                 l->actions[i] = action_find(p, params->action_names[i]);
8744
8745                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8746         }
8747
8748         l->default_action = default_action;
8749
8750         if (default_action->st)
8751                 memcpy(l->default_action_data,
8752                        params->default_action_data,
8753                        default_action->st->n_bits / 8);
8754
8755         l->n_actions = params->n_actions;
8756
8757         l->default_action_is_const = params->default_action_is_const;
8758
8759         l->action_data_size_max = action_data_size_max;
8760
8761         l->size = size;
8762
8763         l->timeout = timeout;
8764
8765         l->id = p->n_learners;
8766
8767         /* Node add to tailq. */
8768         TAILQ_INSERT_TAIL(&p->learners, l, node);
8769         p->n_learners++;
8770
8771         return 0;
8772
8773 nomem:
8774         if (!l)
8775                 return -ENOMEM;
8776
8777         free(l->action_arg);
8778         free(l->actions);
8779         free(l->fields);
8780         free(l);
8781
8782         return -ENOMEM;
8783 }
8784
8785 static void
8786 learner_params_free(struct rte_swx_table_learner_params *params)
8787 {
8788         if (!params)
8789                 return;
8790
8791         free(params->key_mask0);
8792
8793         free(params);
8794 }
8795
8796 static struct rte_swx_table_learner_params *
8797 learner_params_get(struct learner *l)
8798 {
8799         struct rte_swx_table_learner_params *params = NULL;
8800         struct field *first, *last;
8801         uint32_t i;
8802
8803         /* Memory allocation. */
8804         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8805         if (!params)
8806                 goto error;
8807
8808         /* Find first (smallest offset) and last (biggest offset) match fields. */
8809         first = l->fields[0];
8810         last = l->fields[0];
8811
8812         for (i = 0; i < l->n_fields; i++) {
8813                 struct field *f = l->fields[i];
8814
8815                 if (f->offset < first->offset)
8816                         first = f;
8817
8818                 if (f->offset > last->offset)
8819                         last = f;
8820         }
8821
8822         /* Key offset and size. */
8823         params->key_offset = first->offset / 8;
8824         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8825
8826         /* Memory allocation. */
8827         params->key_mask0 = calloc(1, params->key_size);
8828         if (!params->key_mask0)
8829                 goto error;
8830
8831         /* Key mask. */
8832         for (i = 0; i < l->n_fields; i++) {
8833                 struct field *f = l->fields[i];
8834                 uint32_t start = (f->offset - first->offset) / 8;
8835                 size_t size = f->n_bits / 8;
8836
8837                 memset(&params->key_mask0[start], 0xFF, size);
8838         }
8839
8840         /* Action data size. */
8841         params->action_data_size = l->action_data_size_max;
8842
8843         /* Maximum number of keys. */
8844         params->n_keys_max = l->size;
8845
8846         /* Timeout. */
8847         params->key_timeout = l->timeout;
8848
8849         return params;
8850
8851 error:
8852         learner_params_free(params);
8853         return NULL;
8854 }
8855
8856 static void
8857 learner_build_free(struct rte_swx_pipeline *p)
8858 {
8859         uint32_t i;
8860
8861         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8862                 struct thread *t = &p->threads[i];
8863                 uint32_t j;
8864
8865                 if (!t->learners)
8866                         continue;
8867
8868                 for (j = 0; j < p->n_learners; j++) {
8869                         struct learner_runtime *r = &t->learners[j];
8870
8871                         free(r->mailbox);
8872                         free(r->action_data);
8873                 }
8874
8875                 free(t->learners);
8876                 t->learners = NULL;
8877         }
8878
8879         if (p->learner_stats) {
8880                 for (i = 0; i < p->n_learners; i++)
8881                         free(p->learner_stats[i].n_pkts_action);
8882
8883                 free(p->learner_stats);
8884         }
8885 }
8886
8887 static int
8888 learner_build(struct rte_swx_pipeline *p)
8889 {
8890         uint32_t i;
8891         int status = 0;
8892
8893         /* Per pipeline: learner statistics. */
8894         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8895         CHECK(p->learner_stats, ENOMEM);
8896
8897         for (i = 0; i < p->n_learners; i++) {
8898                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8899                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8900         }
8901
8902         /* Per thread: learner run-time. */
8903         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8904                 struct thread *t = &p->threads[i];
8905                 struct learner *l;
8906
8907                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8908                 if (!t->learners) {
8909                         status = -ENOMEM;
8910                         goto error;
8911                 }
8912
8913                 TAILQ_FOREACH(l, &p->learners, node) {
8914                         struct learner_runtime *r = &t->learners[l->id];
8915                         uint64_t size;
8916                         uint32_t j;
8917
8918                         /* r->mailbox. */
8919                         size = rte_swx_table_learner_mailbox_size_get();
8920                         if (size) {
8921                                 r->mailbox = calloc(1, size);
8922                                 if (!r->mailbox) {
8923                                         status = -ENOMEM;
8924                                         goto error;
8925                                 }
8926                         }
8927
8928                         /* r->key. */
8929                         r->key = l->header ?
8930                                 &t->structs[l->header->struct_id] :
8931                                 &t->structs[p->metadata_struct_id];
8932
8933                         /* r->action_data. */
8934                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8935                         if (!r->action_data) {
8936                                 status = -ENOMEM;
8937                                 goto error;
8938                         }
8939
8940                         for (j = 0; j < l->n_actions; j++) {
8941                                 struct action *a = l->actions[j];
8942                                 struct field *mf = l->action_arg[j];
8943                                 uint8_t *m = t->structs[p->metadata_struct_id];
8944
8945                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8946                         }
8947                 }
8948         }
8949
8950         return 0;
8951
8952 error:
8953         learner_build_free(p);
8954         return status;
8955 }
8956
8957 static void
8958 learner_free(struct rte_swx_pipeline *p)
8959 {
8960         learner_build_free(p);
8961
8962         /* Learner tables. */
8963         for ( ; ; ) {
8964                 struct learner *l;
8965
8966                 l = TAILQ_FIRST(&p->learners);
8967                 if (!l)
8968                         break;
8969
8970                 TAILQ_REMOVE(&p->learners, l, node);
8971                 free(l->fields);
8972                 free(l->actions);
8973                 free(l->action_arg);
8974                 free(l->default_action_data);
8975                 free(l);
8976         }
8977 }
8978
8979 /*
8980  * Table state.
8981  */
8982 static int
8983 table_state_build(struct rte_swx_pipeline *p)
8984 {
8985         struct table *table;
8986         struct selector *s;
8987         struct learner *l;
8988
8989         p->table_state = calloc(p->n_tables + p->n_selectors,
8990                                 sizeof(struct rte_swx_table_state));
8991         CHECK(p->table_state, ENOMEM);
8992
8993         TAILQ_FOREACH(table, &p->tables, node) {
8994                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8995
8996                 if (table->type) {
8997                         struct rte_swx_table_params *params;
8998
8999                         /* ts->obj. */
9000                         params = table_params_get(table);
9001                         CHECK(params, ENOMEM);
9002
9003                         ts->obj = table->type->ops.create(params,
9004                                 NULL,
9005                                 table->args,
9006                                 p->numa_node);
9007
9008                         table_params_free(params);
9009                         CHECK(ts->obj, ENODEV);
9010                 }
9011
9012                 /* ts->default_action_data. */
9013                 if (table->action_data_size_max) {
9014                         ts->default_action_data =
9015                                 malloc(table->action_data_size_max);
9016                         CHECK(ts->default_action_data, ENOMEM);
9017
9018                         memcpy(ts->default_action_data,
9019                                table->default_action_data,
9020                                table->action_data_size_max);
9021                 }
9022
9023                 /* ts->default_action_id. */
9024                 ts->default_action_id = table->default_action->id;
9025         }
9026
9027         TAILQ_FOREACH(s, &p->selectors, node) {
9028                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9029                 struct rte_swx_table_selector_params *params;
9030
9031                 /* ts->obj. */
9032                 params = selector_table_params_get(s);
9033                 CHECK(params, ENOMEM);
9034
9035                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9036
9037                 selector_params_free(params);
9038                 CHECK(ts->obj, ENODEV);
9039         }
9040
9041         TAILQ_FOREACH(l, &p->learners, node) {
9042                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9043                         p->n_selectors + l->id];
9044                 struct rte_swx_table_learner_params *params;
9045
9046                 /* ts->obj. */
9047                 params = learner_params_get(l);
9048                 CHECK(params, ENOMEM);
9049
9050                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9051                 learner_params_free(params);
9052                 CHECK(ts->obj, ENODEV);
9053
9054                 /* ts->default_action_data. */
9055                 if (l->action_data_size_max) {
9056                         ts->default_action_data = malloc(l->action_data_size_max);
9057                         CHECK(ts->default_action_data, ENOMEM);
9058
9059                         memcpy(ts->default_action_data,
9060                                l->default_action_data,
9061                                l->action_data_size_max);
9062                 }
9063
9064                 /* ts->default_action_id. */
9065                 ts->default_action_id = l->default_action->id;
9066         }
9067
9068         return 0;
9069 }
9070
9071 static void
9072 table_state_build_free(struct rte_swx_pipeline *p)
9073 {
9074         uint32_t i;
9075
9076         if (!p->table_state)
9077                 return;
9078
9079         for (i = 0; i < p->n_tables; i++) {
9080                 struct rte_swx_table_state *ts = &p->table_state[i];
9081                 struct table *table = table_find_by_id(p, i);
9082
9083                 /* ts->obj. */
9084                 if (table->type && ts->obj)
9085                         table->type->ops.free(ts->obj);
9086
9087                 /* ts->default_action_data. */
9088                 free(ts->default_action_data);
9089         }
9090
9091         for (i = 0; i < p->n_selectors; i++) {
9092                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9093
9094                 /* ts->obj. */
9095                 if (ts->obj)
9096                         rte_swx_table_selector_free(ts->obj);
9097         }
9098
9099         for (i = 0; i < p->n_learners; i++) {
9100                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9101
9102                 /* ts->obj. */
9103                 if (ts->obj)
9104                         rte_swx_table_learner_free(ts->obj);
9105
9106                 /* ts->default_action_data. */
9107                 free(ts->default_action_data);
9108         }
9109
9110         free(p->table_state);
9111         p->table_state = NULL;
9112 }
9113
9114 static void
9115 table_state_free(struct rte_swx_pipeline *p)
9116 {
9117         table_state_build_free(p);
9118 }
9119
9120 /*
9121  * Register array.
9122  */
9123 static struct regarray *
9124 regarray_find(struct rte_swx_pipeline *p, const char *name)
9125 {
9126         struct regarray *elem;
9127
9128         TAILQ_FOREACH(elem, &p->regarrays, node)
9129                 if (!strcmp(elem->name, name))
9130                         return elem;
9131
9132         return NULL;
9133 }
9134
9135 static struct regarray *
9136 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9137 {
9138         struct regarray *elem = NULL;
9139
9140         TAILQ_FOREACH(elem, &p->regarrays, node)
9141                 if (elem->id == id)
9142                         return elem;
9143
9144         return NULL;
9145 }
9146
9147 int
9148 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9149                               const char *name,
9150                               uint32_t size,
9151                               uint64_t init_val)
9152 {
9153         struct regarray *r;
9154
9155         CHECK(p, EINVAL);
9156
9157         CHECK_NAME(name, EINVAL);
9158         CHECK(!regarray_find(p, name), EEXIST);
9159
9160         CHECK(size, EINVAL);
9161         size = rte_align32pow2(size);
9162
9163         /* Memory allocation. */
9164         r = calloc(1, sizeof(struct regarray));
9165         CHECK(r, ENOMEM);
9166
9167         /* Node initialization. */
9168         strcpy(r->name, name);
9169         r->init_val = init_val;
9170         r->size = size;
9171         r->id = p->n_regarrays;
9172
9173         /* Node add to tailq. */
9174         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9175         p->n_regarrays++;
9176
9177         return 0;
9178 }
9179
9180 static int
9181 regarray_build(struct rte_swx_pipeline *p)
9182 {
9183         struct regarray *regarray;
9184
9185         if (!p->n_regarrays)
9186                 return 0;
9187
9188         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9189         CHECK(p->regarray_runtime, ENOMEM);
9190
9191         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9192                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9193                 uint32_t i;
9194
9195                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9196                                          RTE_CACHE_LINE_SIZE,
9197                                          p->numa_node);
9198                 CHECK(r->regarray, ENOMEM);
9199
9200                 if (regarray->init_val)
9201                         for (i = 0; i < regarray->size; i++)
9202                                 r->regarray[i] = regarray->init_val;
9203
9204                 r->size_mask = regarray->size - 1;
9205         }
9206
9207         return 0;
9208 }
9209
9210 static void
9211 regarray_build_free(struct rte_swx_pipeline *p)
9212 {
9213         uint32_t i;
9214
9215         if (!p->regarray_runtime)
9216                 return;
9217
9218         for (i = 0; i < p->n_regarrays; i++) {
9219                 struct regarray *regarray = regarray_find_by_id(p, i);
9220                 struct regarray_runtime *r = &p->regarray_runtime[i];
9221
9222                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9223         }
9224
9225         free(p->regarray_runtime);
9226         p->regarray_runtime = NULL;
9227 }
9228
9229 static void
9230 regarray_free(struct rte_swx_pipeline *p)
9231 {
9232         regarray_build_free(p);
9233
9234         for ( ; ; ) {
9235                 struct regarray *elem;
9236
9237                 elem = TAILQ_FIRST(&p->regarrays);
9238                 if (!elem)
9239                         break;
9240
9241                 TAILQ_REMOVE(&p->regarrays, elem, node);
9242                 free(elem);
9243         }
9244 }
9245
9246 /*
9247  * Meter array.
9248  */
9249 static struct meter_profile *
9250 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9251 {
9252         struct meter_profile *elem;
9253
9254         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9255                 if (!strcmp(elem->name, name))
9256                         return elem;
9257
9258         return NULL;
9259 }
9260
9261 static struct metarray *
9262 metarray_find(struct rte_swx_pipeline *p, const char *name)
9263 {
9264         struct metarray *elem;
9265
9266         TAILQ_FOREACH(elem, &p->metarrays, node)
9267                 if (!strcmp(elem->name, name))
9268                         return elem;
9269
9270         return NULL;
9271 }
9272
9273 static struct metarray *
9274 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9275 {
9276         struct metarray *elem = NULL;
9277
9278         TAILQ_FOREACH(elem, &p->metarrays, node)
9279                 if (elem->id == id)
9280                         return elem;
9281
9282         return NULL;
9283 }
9284
9285 int
9286 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9287                                  const char *name,
9288                                  uint32_t size)
9289 {
9290         struct metarray *m;
9291
9292         CHECK(p, EINVAL);
9293
9294         CHECK_NAME(name, EINVAL);
9295         CHECK(!metarray_find(p, name), EEXIST);
9296
9297         CHECK(size, EINVAL);
9298         size = rte_align32pow2(size);
9299
9300         /* Memory allocation. */
9301         m = calloc(1, sizeof(struct metarray));
9302         CHECK(m, ENOMEM);
9303
9304         /* Node initialization. */
9305         strcpy(m->name, name);
9306         m->size = size;
9307         m->id = p->n_metarrays;
9308
9309         /* Node add to tailq. */
9310         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9311         p->n_metarrays++;
9312
9313         return 0;
9314 }
9315
9316 struct meter_profile meter_profile_default = {
9317         .node = {0},
9318         .name = "",
9319         .params = {0},
9320
9321         .profile = {
9322                 .cbs = 10000,
9323                 .pbs = 10000,
9324                 .cir_period = 1,
9325                 .cir_bytes_per_period = 1,
9326                 .pir_period = 1,
9327                 .pir_bytes_per_period = 1,
9328         },
9329
9330         .n_users = 0,
9331 };
9332
9333 static void
9334 meter_init(struct meter *m)
9335 {
9336         memset(m, 0, sizeof(struct meter));
9337         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9338         m->profile = &meter_profile_default;
9339         m->color_mask = RTE_COLOR_GREEN;
9340
9341         meter_profile_default.n_users++;
9342 }
9343
9344 static int
9345 metarray_build(struct rte_swx_pipeline *p)
9346 {
9347         struct metarray *m;
9348
9349         if (!p->n_metarrays)
9350                 return 0;
9351
9352         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9353         CHECK(p->metarray_runtime, ENOMEM);
9354
9355         TAILQ_FOREACH(m, &p->metarrays, node) {
9356                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9357                 uint32_t i;
9358
9359                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9360                                          RTE_CACHE_LINE_SIZE,
9361                                          p->numa_node);
9362                 CHECK(r->metarray, ENOMEM);
9363
9364                 for (i = 0; i < m->size; i++)
9365                         meter_init(&r->metarray[i]);
9366
9367                 r->size_mask = m->size - 1;
9368         }
9369
9370         return 0;
9371 }
9372
9373 static void
9374 metarray_build_free(struct rte_swx_pipeline *p)
9375 {
9376         uint32_t i;
9377
9378         if (!p->metarray_runtime)
9379                 return;
9380
9381         for (i = 0; i < p->n_metarrays; i++) {
9382                 struct metarray *m = metarray_find_by_id(p, i);
9383                 struct metarray_runtime *r = &p->metarray_runtime[i];
9384
9385                 env_free(r->metarray, m->size * sizeof(struct meter));
9386         }
9387
9388         free(p->metarray_runtime);
9389         p->metarray_runtime = NULL;
9390 }
9391
9392 static void
9393 metarray_free(struct rte_swx_pipeline *p)
9394 {
9395         metarray_build_free(p);
9396
9397         /* Meter arrays. */
9398         for ( ; ; ) {
9399                 struct metarray *elem;
9400
9401                 elem = TAILQ_FIRST(&p->metarrays);
9402                 if (!elem)
9403                         break;
9404
9405                 TAILQ_REMOVE(&p->metarrays, elem, node);
9406                 free(elem);
9407         }
9408
9409         /* Meter profiles. */
9410         for ( ; ; ) {
9411                 struct meter_profile *elem;
9412
9413                 elem = TAILQ_FIRST(&p->meter_profiles);
9414                 if (!elem)
9415                         break;
9416
9417                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9418                 free(elem);
9419         }
9420 }
9421
9422 /*
9423  * Pipeline.
9424  */
9425 int
9426 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9427 {
9428         struct rte_swx_pipeline *pipeline;
9429
9430         /* Check input parameters. */
9431         CHECK(p, EINVAL);
9432
9433         /* Memory allocation. */
9434         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9435         CHECK(pipeline, ENOMEM);
9436
9437         /* Initialization. */
9438         TAILQ_INIT(&pipeline->struct_types);
9439         TAILQ_INIT(&pipeline->port_in_types);
9440         TAILQ_INIT(&pipeline->ports_in);
9441         TAILQ_INIT(&pipeline->port_out_types);
9442         TAILQ_INIT(&pipeline->ports_out);
9443         TAILQ_INIT(&pipeline->extern_types);
9444         TAILQ_INIT(&pipeline->extern_objs);
9445         TAILQ_INIT(&pipeline->extern_funcs);
9446         TAILQ_INIT(&pipeline->headers);
9447         TAILQ_INIT(&pipeline->actions);
9448         TAILQ_INIT(&pipeline->table_types);
9449         TAILQ_INIT(&pipeline->tables);
9450         TAILQ_INIT(&pipeline->selectors);
9451         TAILQ_INIT(&pipeline->learners);
9452         TAILQ_INIT(&pipeline->regarrays);
9453         TAILQ_INIT(&pipeline->meter_profiles);
9454         TAILQ_INIT(&pipeline->metarrays);
9455
9456         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9457         pipeline->numa_node = numa_node;
9458
9459         *p = pipeline;
9460         return 0;
9461 }
9462
9463 void
9464 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9465 {
9466         if (!p)
9467                 return;
9468
9469         free(p->instructions);
9470
9471         metarray_free(p);
9472         regarray_free(p);
9473         table_state_free(p);
9474         learner_free(p);
9475         selector_free(p);
9476         table_free(p);
9477         action_free(p);
9478         metadata_free(p);
9479         header_free(p);
9480         extern_func_free(p);
9481         extern_obj_free(p);
9482         port_out_free(p);
9483         port_in_free(p);
9484         struct_free(p);
9485
9486         free(p);
9487 }
9488
9489 int
9490 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9491                                      const char **instructions,
9492                                      uint32_t n_instructions)
9493 {
9494         int err;
9495         uint32_t i;
9496
9497         err = instruction_config(p, NULL, instructions, n_instructions);
9498         if (err)
9499                 return err;
9500
9501         /* Thread instruction pointer reset. */
9502         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9503                 struct thread *t = &p->threads[i];
9504
9505                 thread_ip_reset(p, t);
9506         }
9507
9508         return 0;
9509 }
9510
9511 int
9512 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9513 {
9514         int status;
9515
9516         CHECK(p, EINVAL);
9517         CHECK(p->build_done == 0, EEXIST);
9518
9519         status = port_in_build(p);
9520         if (status)
9521                 goto error;
9522
9523         status = port_out_build(p);
9524         if (status)
9525                 goto error;
9526
9527         status = struct_build(p);
9528         if (status)
9529                 goto error;
9530
9531         status = extern_obj_build(p);
9532         if (status)
9533                 goto error;
9534
9535         status = extern_func_build(p);
9536         if (status)
9537                 goto error;
9538
9539         status = header_build(p);
9540         if (status)
9541                 goto error;
9542
9543         status = metadata_build(p);
9544         if (status)
9545                 goto error;
9546
9547         status = action_build(p);
9548         if (status)
9549                 goto error;
9550
9551         status = table_build(p);
9552         if (status)
9553                 goto error;
9554
9555         status = selector_build(p);
9556         if (status)
9557                 goto error;
9558
9559         status = learner_build(p);
9560         if (status)
9561                 goto error;
9562
9563         status = table_state_build(p);
9564         if (status)
9565                 goto error;
9566
9567         status = regarray_build(p);
9568         if (status)
9569                 goto error;
9570
9571         status = metarray_build(p);
9572         if (status)
9573                 goto error;
9574
9575         p->build_done = 1;
9576         return 0;
9577
9578 error:
9579         metarray_build_free(p);
9580         regarray_build_free(p);
9581         table_state_build_free(p);
9582         learner_build_free(p);
9583         selector_build_free(p);
9584         table_build_free(p);
9585         action_build_free(p);
9586         metadata_build_free(p);
9587         header_build_free(p);
9588         extern_func_build_free(p);
9589         extern_obj_build_free(p);
9590         port_out_build_free(p);
9591         port_in_build_free(p);
9592         struct_build_free(p);
9593
9594         return status;
9595 }
9596
9597 void
9598 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9599 {
9600         uint32_t i;
9601
9602         for (i = 0; i < n_instructions; i++)
9603                 instr_exec(p);
9604 }
9605
9606 void
9607 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9608 {
9609         uint32_t i;
9610
9611         for (i = 0; i < p->n_ports_out; i++) {
9612                 struct port_out_runtime *port = &p->out[i];
9613
9614                 if (port->flush)
9615                         port->flush(port->obj);
9616         }
9617 }
9618
9619 /*
9620  * Control.
9621  */
9622 int
9623 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9624                               struct rte_swx_ctl_pipeline_info *pipeline)
9625 {
9626         struct action *action;
9627         struct table *table;
9628         uint32_t n_actions = 0, n_tables = 0;
9629
9630         if (!p || !pipeline)
9631                 return -EINVAL;
9632
9633         TAILQ_FOREACH(action, &p->actions, node)
9634                 n_actions++;
9635
9636         TAILQ_FOREACH(table, &p->tables, node)
9637                 n_tables++;
9638
9639         pipeline->n_ports_in = p->n_ports_in;
9640         pipeline->n_ports_out = p->n_ports_out;
9641         pipeline->n_actions = n_actions;
9642         pipeline->n_tables = n_tables;
9643         pipeline->n_selectors = p->n_selectors;
9644         pipeline->n_learners = p->n_learners;
9645         pipeline->n_regarrays = p->n_regarrays;
9646         pipeline->n_metarrays = p->n_metarrays;
9647
9648         return 0;
9649 }
9650
9651 int
9652 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9653 {
9654         if (!p || !numa_node)
9655                 return -EINVAL;
9656
9657         *numa_node = p->numa_node;
9658         return 0;
9659 }
9660
9661 int
9662 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9663                             uint32_t action_id,
9664                             struct rte_swx_ctl_action_info *action)
9665 {
9666         struct action *a = NULL;
9667
9668         if (!p || (action_id >= p->n_actions) || !action)
9669                 return -EINVAL;
9670
9671         a = action_find_by_id(p, action_id);
9672         if (!a)
9673                 return -EINVAL;
9674
9675         strcpy(action->name, a->name);
9676         action->n_args = a->st ? a->st->n_fields : 0;
9677         return 0;
9678 }
9679
9680 int
9681 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9682                                 uint32_t action_id,
9683                                 uint32_t action_arg_id,
9684                                 struct rte_swx_ctl_action_arg_info *action_arg)
9685 {
9686         struct action *a = NULL;
9687         struct field *arg = NULL;
9688
9689         if (!p || (action_id >= p->n_actions) || !action_arg)
9690                 return -EINVAL;
9691
9692         a = action_find_by_id(p, action_id);
9693         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9694                 return -EINVAL;
9695
9696         arg = &a->st->fields[action_arg_id];
9697         strcpy(action_arg->name, arg->name);
9698         action_arg->n_bits = arg->n_bits;
9699         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9700
9701         return 0;
9702 }
9703
9704 int
9705 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9706                            uint32_t table_id,
9707                            struct rte_swx_ctl_table_info *table)
9708 {
9709         struct table *t = NULL;
9710
9711         if (!p || !table)
9712                 return -EINVAL;
9713
9714         t = table_find_by_id(p, table_id);
9715         if (!t)
9716                 return -EINVAL;
9717
9718         strcpy(table->name, t->name);
9719         strcpy(table->args, t->args);
9720         table->n_match_fields = t->n_fields;
9721         table->n_actions = t->n_actions;
9722         table->default_action_is_const = t->default_action_is_const;
9723         table->size = t->size;
9724         return 0;
9725 }
9726
9727 int
9728 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9729         uint32_t table_id,
9730         uint32_t match_field_id,
9731         struct rte_swx_ctl_table_match_field_info *match_field)
9732 {
9733         struct table *t;
9734         struct match_field *f;
9735
9736         if (!p || (table_id >= p->n_tables) || !match_field)
9737                 return -EINVAL;
9738
9739         t = table_find_by_id(p, table_id);
9740         if (!t || (match_field_id >= t->n_fields))
9741                 return -EINVAL;
9742
9743         f = &t->fields[match_field_id];
9744         match_field->match_type = f->match_type;
9745         match_field->is_header = t->header ? 1 : 0;
9746         match_field->n_bits = f->field->n_bits;
9747         match_field->offset = f->field->offset;
9748
9749         return 0;
9750 }
9751
9752 int
9753 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9754         uint32_t table_id,
9755         uint32_t table_action_id,
9756         struct rte_swx_ctl_table_action_info *table_action)
9757 {
9758         struct table *t;
9759
9760         if (!p || (table_id >= p->n_tables) || !table_action)
9761                 return -EINVAL;
9762
9763         t = table_find_by_id(p, table_id);
9764         if (!t || (table_action_id >= t->n_actions))
9765                 return -EINVAL;
9766
9767         table_action->action_id = t->actions[table_action_id]->id;
9768
9769         return 0;
9770 }
9771
9772 int
9773 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9774                           uint32_t table_id,
9775                           struct rte_swx_table_ops *table_ops,
9776                           int *is_stub)
9777 {
9778         struct table *t;
9779
9780         if (!p || (table_id >= p->n_tables))
9781                 return -EINVAL;
9782
9783         t = table_find_by_id(p, table_id);
9784         if (!t)
9785                 return -EINVAL;
9786
9787         if (t->type) {
9788                 if (table_ops)
9789                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9790                 *is_stub = 0;
9791         } else {
9792                 *is_stub = 1;
9793         }
9794
9795         return 0;
9796 }
9797
9798 int
9799 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9800                               uint32_t selector_id,
9801                               struct rte_swx_ctl_selector_info *selector)
9802 {
9803         struct selector *s = NULL;
9804
9805         if (!p || !selector)
9806                 return -EINVAL;
9807
9808         s = selector_find_by_id(p, selector_id);
9809         if (!s)
9810                 return -EINVAL;
9811
9812         strcpy(selector->name, s->name);
9813
9814         selector->n_selector_fields = s->n_selector_fields;
9815         selector->n_groups_max = s->n_groups_max;
9816         selector->n_members_per_group_max = s->n_members_per_group_max;
9817
9818         return 0;
9819 }
9820
9821 int
9822 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9823          uint32_t selector_id,
9824          struct rte_swx_ctl_table_match_field_info *field)
9825 {
9826         struct selector *s;
9827
9828         if (!p || (selector_id >= p->n_selectors) || !field)
9829                 return -EINVAL;
9830
9831         s = selector_find_by_id(p, selector_id);
9832         if (!s)
9833                 return -EINVAL;
9834
9835         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9836         field->is_header = 0;
9837         field->n_bits = s->group_id_field->n_bits;
9838         field->offset = s->group_id_field->offset;
9839
9840         return 0;
9841 }
9842
9843 int
9844 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9845          uint32_t selector_id,
9846          uint32_t selector_field_id,
9847          struct rte_swx_ctl_table_match_field_info *field)
9848 {
9849         struct selector *s;
9850         struct field *f;
9851
9852         if (!p || (selector_id >= p->n_selectors) || !field)
9853                 return -EINVAL;
9854
9855         s = selector_find_by_id(p, selector_id);
9856         if (!s || (selector_field_id >= s->n_selector_fields))
9857                 return -EINVAL;
9858
9859         f = s->selector_fields[selector_field_id];
9860         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9861         field->is_header = s->selector_header ? 1 : 0;
9862         field->n_bits = f->n_bits;
9863         field->offset = f->offset;
9864
9865         return 0;
9866 }
9867
9868 int
9869 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9870          uint32_t selector_id,
9871          struct rte_swx_ctl_table_match_field_info *field)
9872 {
9873         struct selector *s;
9874
9875         if (!p || (selector_id >= p->n_selectors) || !field)
9876                 return -EINVAL;
9877
9878         s = selector_find_by_id(p, selector_id);
9879         if (!s)
9880                 return -EINVAL;
9881
9882         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9883         field->is_header = 0;
9884         field->n_bits = s->member_id_field->n_bits;
9885         field->offset = s->member_id_field->offset;
9886
9887         return 0;
9888 }
9889
9890 int
9891 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9892                              uint32_t learner_id,
9893                              struct rte_swx_ctl_learner_info *learner)
9894 {
9895         struct learner *l = NULL;
9896
9897         if (!p || !learner)
9898                 return -EINVAL;
9899
9900         l = learner_find_by_id(p, learner_id);
9901         if (!l)
9902                 return -EINVAL;
9903
9904         strcpy(learner->name, l->name);
9905
9906         learner->n_match_fields = l->n_fields;
9907         learner->n_actions = l->n_actions;
9908         learner->default_action_is_const = l->default_action_is_const;
9909         learner->size = l->size;
9910
9911         return 0;
9912 }
9913
9914 int
9915 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9916                                          uint32_t learner_id,
9917                                          uint32_t match_field_id,
9918                                          struct rte_swx_ctl_table_match_field_info *match_field)
9919 {
9920         struct learner *l;
9921         struct field *f;
9922
9923         if (!p || (learner_id >= p->n_learners) || !match_field)
9924                 return -EINVAL;
9925
9926         l = learner_find_by_id(p, learner_id);
9927         if (!l || (match_field_id >= l->n_fields))
9928                 return -EINVAL;
9929
9930         f = l->fields[match_field_id];
9931         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9932         match_field->is_header = l->header ? 1 : 0;
9933         match_field->n_bits = f->n_bits;
9934         match_field->offset = f->offset;
9935
9936         return 0;
9937 }
9938
9939 int
9940 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9941                                     uint32_t learner_id,
9942                                     uint32_t learner_action_id,
9943                                     struct rte_swx_ctl_table_action_info *learner_action)
9944 {
9945         struct learner *l;
9946
9947         if (!p || (learner_id >= p->n_learners) || !learner_action)
9948                 return -EINVAL;
9949
9950         l = learner_find_by_id(p, learner_id);
9951         if (!l || (learner_action_id >= l->n_actions))
9952                 return -EINVAL;
9953
9954         learner_action->action_id = l->actions[learner_action_id]->id;
9955
9956         return 0;
9957 }
9958
9959 int
9960 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9961                                  struct rte_swx_table_state **table_state)
9962 {
9963         if (!p || !table_state || !p->build_done)
9964                 return -EINVAL;
9965
9966         *table_state = p->table_state;
9967         return 0;
9968 }
9969
9970 int
9971 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9972                                  struct rte_swx_table_state *table_state)
9973 {
9974         if (!p || !table_state || !p->build_done)
9975                 return -EINVAL;
9976
9977         p->table_state = table_state;
9978         return 0;
9979 }
9980
9981 int
9982 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9983                                         uint32_t port_id,
9984                                         struct rte_swx_port_in_stats *stats)
9985 {
9986         struct port_in *port;
9987
9988         if (!p || !stats)
9989                 return -EINVAL;
9990
9991         port = port_in_find(p, port_id);
9992         if (!port)
9993                 return -EINVAL;
9994
9995         port->type->ops.stats_read(port->obj, stats);
9996         return 0;
9997 }
9998
9999 int
10000 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10001                                          uint32_t port_id,
10002                                          struct rte_swx_port_out_stats *stats)
10003 {
10004         struct port_out *port;
10005
10006         if (!p || !stats)
10007                 return -EINVAL;
10008
10009         port = port_out_find(p, port_id);
10010         if (!port)
10011                 return -EINVAL;
10012
10013         port->type->ops.stats_read(port->obj, stats);
10014         return 0;
10015 }
10016
10017 int
10018 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10019                                       const char *table_name,
10020                                       struct rte_swx_table_stats *stats)
10021 {
10022         struct table *table;
10023         struct table_statistics *table_stats;
10024
10025         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10026                 return -EINVAL;
10027
10028         table = table_find(p, table_name);
10029         if (!table)
10030                 return -EINVAL;
10031
10032         table_stats = &p->table_stats[table->id];
10033
10034         memcpy(stats->n_pkts_action,
10035                table_stats->n_pkts_action,
10036                p->n_actions * sizeof(uint64_t));
10037
10038         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10039         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10040
10041         return 0;
10042 }
10043
10044 int
10045 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10046         const char *selector_name,
10047         struct rte_swx_pipeline_selector_stats *stats)
10048 {
10049         struct selector *s;
10050
10051         if (!p || !selector_name || !selector_name[0] || !stats)
10052                 return -EINVAL;
10053
10054         s = selector_find(p, selector_name);
10055         if (!s)
10056                 return -EINVAL;
10057
10058         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10059
10060         return 0;
10061 }
10062
10063 int
10064 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10065                                         const char *learner_name,
10066                                         struct rte_swx_learner_stats *stats)
10067 {
10068         struct learner *l;
10069         struct learner_statistics *learner_stats;
10070
10071         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10072                 return -EINVAL;
10073
10074         l = learner_find(p, learner_name);
10075         if (!l)
10076                 return -EINVAL;
10077
10078         learner_stats = &p->learner_stats[l->id];
10079
10080         memcpy(stats->n_pkts_action,
10081                learner_stats->n_pkts_action,
10082                p->n_actions * sizeof(uint64_t));
10083
10084         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10085         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10086
10087         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10088         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10089
10090         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10091
10092         return 0;
10093 }
10094
10095 int
10096 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10097                               uint32_t regarray_id,
10098                               struct rte_swx_ctl_regarray_info *regarray)
10099 {
10100         struct regarray *r;
10101
10102         if (!p || !regarray)
10103                 return -EINVAL;
10104
10105         r = regarray_find_by_id(p, regarray_id);
10106         if (!r)
10107                 return -EINVAL;
10108
10109         strcpy(regarray->name, r->name);
10110         regarray->size = r->size;
10111         return 0;
10112 }
10113
10114 int
10115 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10116                                    const char *regarray_name,
10117                                    uint32_t regarray_index,
10118                                    uint64_t *value)
10119 {
10120         struct regarray *regarray;
10121         struct regarray_runtime *r;
10122
10123         if (!p || !regarray_name || !value)
10124                 return -EINVAL;
10125
10126         regarray = regarray_find(p, regarray_name);
10127         if (!regarray || (regarray_index >= regarray->size))
10128                 return -EINVAL;
10129
10130         r = &p->regarray_runtime[regarray->id];
10131         *value = r->regarray[regarray_index];
10132         return 0;
10133 }
10134
10135 int
10136 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10137                                    const char *regarray_name,
10138                                    uint32_t regarray_index,
10139                                    uint64_t value)
10140 {
10141         struct regarray *regarray;
10142         struct regarray_runtime *r;
10143
10144         if (!p || !regarray_name)
10145                 return -EINVAL;
10146
10147         regarray = regarray_find(p, regarray_name);
10148         if (!regarray || (regarray_index >= regarray->size))
10149                 return -EINVAL;
10150
10151         r = &p->regarray_runtime[regarray->id];
10152         r->regarray[regarray_index] = value;
10153         return 0;
10154 }
10155
10156 int
10157 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10158                               uint32_t metarray_id,
10159                               struct rte_swx_ctl_metarray_info *metarray)
10160 {
10161         struct metarray *m;
10162
10163         if (!p || !metarray)
10164                 return -EINVAL;
10165
10166         m = metarray_find_by_id(p, metarray_id);
10167         if (!m)
10168                 return -EINVAL;
10169
10170         strcpy(metarray->name, m->name);
10171         metarray->size = m->size;
10172         return 0;
10173 }
10174
10175 int
10176 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10177                               const char *name,
10178                               struct rte_meter_trtcm_params *params)
10179 {
10180         struct meter_profile *mp;
10181         int status;
10182
10183         CHECK(p, EINVAL);
10184         CHECK_NAME(name, EINVAL);
10185         CHECK(params, EINVAL);
10186         CHECK(!meter_profile_find(p, name), EEXIST);
10187
10188         /* Node allocation. */
10189         mp = calloc(1, sizeof(struct meter_profile));
10190         CHECK(mp, ENOMEM);
10191
10192         /* Node initialization. */
10193         strcpy(mp->name, name);
10194         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10195         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10196         if (status) {
10197                 free(mp);
10198                 CHECK(0, EINVAL);
10199         }
10200
10201         /* Node add to tailq. */
10202         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10203
10204         return 0;
10205 }
10206
10207 int
10208 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10209                                  const char *name)
10210 {
10211         struct meter_profile *mp;
10212
10213         CHECK(p, EINVAL);
10214         CHECK_NAME(name, EINVAL);
10215
10216         mp = meter_profile_find(p, name);
10217         CHECK(mp, EINVAL);
10218         CHECK(!mp->n_users, EBUSY);
10219
10220         /* Remove node from tailq. */
10221         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10222         free(mp);
10223
10224         return 0;
10225 }
10226
10227 int
10228 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10229                         const char *metarray_name,
10230                         uint32_t metarray_index)
10231 {
10232         struct meter_profile *mp_old;
10233         struct metarray *metarray;
10234         struct metarray_runtime *metarray_runtime;
10235         struct meter *m;
10236
10237         CHECK(p, EINVAL);
10238         CHECK_NAME(metarray_name, EINVAL);
10239
10240         metarray = metarray_find(p, metarray_name);
10241         CHECK(metarray, EINVAL);
10242         CHECK(metarray_index < metarray->size, EINVAL);
10243
10244         metarray_runtime = &p->metarray_runtime[metarray->id];
10245         m = &metarray_runtime->metarray[metarray_index];
10246         mp_old = m->profile;
10247
10248         meter_init(m);
10249
10250         mp_old->n_users--;
10251
10252         return 0;
10253 }
10254
10255 int
10256 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10257                       const char *metarray_name,
10258                       uint32_t metarray_index,
10259                       const char *profile_name)
10260 {
10261         struct meter_profile *mp, *mp_old;
10262         struct metarray *metarray;
10263         struct metarray_runtime *metarray_runtime;
10264         struct meter *m;
10265
10266         CHECK(p, EINVAL);
10267         CHECK_NAME(metarray_name, EINVAL);
10268
10269         metarray = metarray_find(p, metarray_name);
10270         CHECK(metarray, EINVAL);
10271         CHECK(metarray_index < metarray->size, EINVAL);
10272
10273         mp = meter_profile_find(p, profile_name);
10274         CHECK(mp, EINVAL);
10275
10276         metarray_runtime = &p->metarray_runtime[metarray->id];
10277         m = &metarray_runtime->metarray[metarray_index];
10278         mp_old = m->profile;
10279
10280         memset(m, 0, sizeof(struct meter));
10281         rte_meter_trtcm_config(&m->m, &mp->profile);
10282         m->profile = mp;
10283         m->color_mask = RTE_COLORS;
10284
10285         mp->n_users++;
10286         mp_old->n_users--;
10287
10288         return 0;
10289 }
10290
10291 int
10292 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10293                              const char *metarray_name,
10294                              uint32_t metarray_index,
10295                              struct rte_swx_ctl_meter_stats *stats)
10296 {
10297         struct metarray *metarray;
10298         struct metarray_runtime *metarray_runtime;
10299         struct meter *m;
10300
10301         CHECK(p, EINVAL);
10302         CHECK_NAME(metarray_name, EINVAL);
10303
10304         metarray = metarray_find(p, metarray_name);
10305         CHECK(metarray, EINVAL);
10306         CHECK(metarray_index < metarray->size, EINVAL);
10307
10308         CHECK(stats, EINVAL);
10309
10310         metarray_runtime = &p->metarray_runtime[metarray->id];
10311         m = &metarray_runtime->metarray[metarray_index];
10312
10313         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10314         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10315
10316         return 0;
10317 }