lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 static inline void
1521 instr_rx_exec(struct rte_swx_pipeline *p);
1522
1523 static inline void
1524 instr_rx_exec(struct rte_swx_pipeline *p)
1525 {
1526         struct thread *t = &p->threads[p->thread_id];
1527         struct instruction *ip = t->ip;
1528         struct port_in_runtime *port = &p->in[p->port_id];
1529         struct rte_swx_pkt *pkt = &t->pkt;
1530         int pkt_received;
1531
1532         /* Packet. */
1533         pkt_received = port->pkt_rx(port->obj, pkt);
1534         t->ptr = &pkt->pkt[pkt->offset];
1535         rte_prefetch0(t->ptr);
1536
1537         TRACE("[Thread %2u] rx %s from port %u\n",
1538               p->thread_id,
1539               pkt_received ? "1 pkt" : "0 pkts",
1540               p->port_id);
1541
1542         /* Headers. */
1543         t->valid_headers = 0;
1544         t->n_headers_out = 0;
1545
1546         /* Meta-data. */
1547         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1548
1549         /* Tables. */
1550         t->table_state = p->table_state;
1551
1552         /* Thread. */
1553         pipeline_port_inc(p);
1554         thread_ip_inc_cond(t, pkt_received);
1555         thread_yield(p);
1556 }
1557
1558 /*
1559  * tx.
1560  */
1561 static int
1562 instr_tx_translate(struct rte_swx_pipeline *p,
1563                    struct action *action __rte_unused,
1564                    char **tokens,
1565                    int n_tokens,
1566                    struct instruction *instr,
1567                    struct instruction_data *data __rte_unused)
1568 {
1569         char *port = tokens[1];
1570         struct field *f;
1571         uint32_t port_val;
1572
1573         CHECK(n_tokens == 2, EINVAL);
1574
1575         f = metadata_field_parse(p, port);
1576         if (f) {
1577                 instr->type = INSTR_TX;
1578                 instr->io.io.offset = f->offset / 8;
1579                 instr->io.io.n_bits = f->n_bits;
1580                 return 0;
1581         }
1582
1583         /* TX_I. */
1584         port_val = strtoul(port, &port, 0);
1585         CHECK(!port[0], EINVAL);
1586
1587         instr->type = INSTR_TX_I;
1588         instr->io.io.val = port_val;
1589         return 0;
1590 }
1591
1592 static int
1593 instr_drop_translate(struct rte_swx_pipeline *p,
1594                      struct action *action __rte_unused,
1595                      char **tokens __rte_unused,
1596                      int n_tokens,
1597                      struct instruction *instr,
1598                      struct instruction_data *data __rte_unused)
1599 {
1600         CHECK(n_tokens == 1, EINVAL);
1601
1602         /* TX_I. */
1603         instr->type = INSTR_TX_I;
1604         instr->io.io.val = p->n_ports_out - 1;
1605         return 0;
1606 }
1607
1608 static inline void
1609 emit_handler(struct thread *t)
1610 {
1611         struct header_out_runtime *h0 = &t->headers_out[0];
1612         struct header_out_runtime *h1 = &t->headers_out[1];
1613         uint32_t offset = 0, i;
1614
1615         /* No header change or header decapsulation. */
1616         if ((t->n_headers_out == 1) &&
1617             (h0->ptr + h0->n_bytes == t->ptr)) {
1618                 TRACE("Emit handler: no header change or header decap.\n");
1619
1620                 t->pkt.offset -= h0->n_bytes;
1621                 t->pkt.length += h0->n_bytes;
1622
1623                 return;
1624         }
1625
1626         /* Header encapsulation (optionally, with prior header decasulation). */
1627         if ((t->n_headers_out == 2) &&
1628             (h1->ptr + h1->n_bytes == t->ptr) &&
1629             (h0->ptr == h0->ptr0)) {
1630                 uint32_t offset;
1631
1632                 TRACE("Emit handler: header encapsulation.\n");
1633
1634                 offset = h0->n_bytes + h1->n_bytes;
1635                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1636                 t->pkt.offset -= offset;
1637                 t->pkt.length += offset;
1638
1639                 return;
1640         }
1641
1642         /* Header insertion. */
1643         /* TBD */
1644
1645         /* Header extraction. */
1646         /* TBD */
1647
1648         /* For any other case. */
1649         TRACE("Emit handler: complex case.\n");
1650
1651         for (i = 0; i < t->n_headers_out; i++) {
1652                 struct header_out_runtime *h = &t->headers_out[i];
1653
1654                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1655                 offset += h->n_bytes;
1656         }
1657
1658         if (offset) {
1659                 memcpy(t->ptr - offset, t->header_out_storage, offset);
1660                 t->pkt.offset -= offset;
1661                 t->pkt.length += offset;
1662         }
1663 }
1664
1665 static inline void
1666 instr_tx_exec(struct rte_swx_pipeline *p);
1667
1668 static inline void
1669 instr_tx_exec(struct rte_swx_pipeline *p)
1670 {
1671         struct thread *t = &p->threads[p->thread_id];
1672         struct instruction *ip = t->ip;
1673         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1674         struct port_out_runtime *port = &p->out[port_id];
1675         struct rte_swx_pkt *pkt = &t->pkt;
1676
1677         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1678               p->thread_id,
1679               (uint32_t)port_id);
1680
1681         /* Headers. */
1682         emit_handler(t);
1683
1684         /* Packet. */
1685         port->pkt_tx(port->obj, pkt);
1686
1687         /* Thread. */
1688         thread_ip_reset(p, t);
1689         instr_rx_exec(p);
1690 }
1691
1692 static inline void
1693 instr_tx_i_exec(struct rte_swx_pipeline *p)
1694 {
1695         struct thread *t = &p->threads[p->thread_id];
1696         struct instruction *ip = t->ip;
1697         uint64_t port_id = ip->io.io.val;
1698         struct port_out_runtime *port = &p->out[port_id];
1699         struct rte_swx_pkt *pkt = &t->pkt;
1700
1701         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1702               p->thread_id,
1703               (uint32_t)port_id);
1704
1705         /* Headers. */
1706         emit_handler(t);
1707
1708         /* Packet. */
1709         port->pkt_tx(port->obj, pkt);
1710
1711         /* Thread. */
1712         thread_ip_reset(p, t);
1713         instr_rx_exec(p);
1714 }
1715
1716 /*
1717  * extract.
1718  */
1719 static int
1720 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1721                             struct action *action,
1722                             char **tokens,
1723                             int n_tokens,
1724                             struct instruction *instr,
1725                             struct instruction_data *data __rte_unused)
1726 {
1727         struct header *h;
1728
1729         CHECK(!action, EINVAL);
1730         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1731
1732         h = header_parse(p, tokens[1]);
1733         CHECK(h, EINVAL);
1734
1735         if (n_tokens == 2) {
1736                 CHECK(!h->st->var_size, EINVAL);
1737
1738                 instr->type = INSTR_HDR_EXTRACT;
1739                 instr->io.hdr.header_id[0] = h->id;
1740                 instr->io.hdr.struct_id[0] = h->struct_id;
1741                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1742         } else {
1743                 struct field *mf;
1744
1745                 CHECK(h->st->var_size, EINVAL);
1746
1747                 mf = metadata_field_parse(p, tokens[2]);
1748                 CHECK(mf, EINVAL);
1749                 CHECK(!mf->var_size, EINVAL);
1750
1751                 instr->type = INSTR_HDR_EXTRACT_M;
1752                 instr->io.io.offset = mf->offset / 8;
1753                 instr->io.io.n_bits = mf->n_bits;
1754                 instr->io.hdr.header_id[0] = h->id;
1755                 instr->io.hdr.struct_id[0] = h->struct_id;
1756                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1757         }
1758
1759         return 0;
1760 }
1761
1762 static int
1763 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1764                               struct action *action,
1765                               char **tokens,
1766                               int n_tokens,
1767                               struct instruction *instr,
1768                               struct instruction_data *data __rte_unused)
1769 {
1770         struct header *h;
1771
1772         CHECK(!action, EINVAL);
1773         CHECK(n_tokens == 2, EINVAL);
1774
1775         h = header_parse(p, tokens[1]);
1776         CHECK(h, EINVAL);
1777         CHECK(!h->st->var_size, EINVAL);
1778
1779         instr->type = INSTR_HDR_LOOKAHEAD;
1780         instr->io.hdr.header_id[0] = h->id;
1781         instr->io.hdr.struct_id[0] = h->struct_id;
1782         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1783
1784         return 0;
1785 }
1786
1787 static inline void
1788 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
1789
1790 static inline void
1791 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
1792 {
1793         struct thread *t = &p->threads[p->thread_id];
1794         struct instruction *ip = t->ip;
1795         uint64_t valid_headers = t->valid_headers;
1796         uint8_t *ptr = t->ptr;
1797         uint32_t offset = t->pkt.offset;
1798         uint32_t length = t->pkt.length;
1799         uint32_t i;
1800
1801         for (i = 0; i < n_extract; i++) {
1802                 uint32_t header_id = ip->io.hdr.header_id[i];
1803                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1804                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1805
1806                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1807                       p->thread_id,
1808                       header_id,
1809                       n_bytes);
1810
1811                 /* Headers. */
1812                 t->structs[struct_id] = ptr;
1813                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1814
1815                 /* Packet. */
1816                 offset += n_bytes;
1817                 length -= n_bytes;
1818                 ptr += n_bytes;
1819         }
1820
1821         /* Headers. */
1822         t->valid_headers = valid_headers;
1823
1824         /* Packet. */
1825         t->pkt.offset = offset;
1826         t->pkt.length = length;
1827         t->ptr = ptr;
1828 }
1829
1830 static inline void
1831 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1832 {
1833         __instr_hdr_extract_exec(p, 1);
1834
1835         /* Thread. */
1836         thread_ip_inc(p);
1837 }
1838
1839 static inline void
1840 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1841 {
1842         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
1843               p->thread_id);
1844
1845         __instr_hdr_extract_exec(p, 2);
1846
1847         /* Thread. */
1848         thread_ip_inc(p);
1849 }
1850
1851 static inline void
1852 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1853 {
1854         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
1855               p->thread_id);
1856
1857         __instr_hdr_extract_exec(p, 3);
1858
1859         /* Thread. */
1860         thread_ip_inc(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1865 {
1866         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
1867               p->thread_id);
1868
1869         __instr_hdr_extract_exec(p, 4);
1870
1871         /* Thread. */
1872         thread_ip_inc(p);
1873 }
1874
1875 static inline void
1876 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1877 {
1878         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
1879               p->thread_id);
1880
1881         __instr_hdr_extract_exec(p, 5);
1882
1883         /* Thread. */
1884         thread_ip_inc(p);
1885 }
1886
1887 static inline void
1888 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1889 {
1890         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
1891               p->thread_id);
1892
1893         __instr_hdr_extract_exec(p, 6);
1894
1895         /* Thread. */
1896         thread_ip_inc(p);
1897 }
1898
1899 static inline void
1900 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1901 {
1902         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
1903               p->thread_id);
1904
1905         __instr_hdr_extract_exec(p, 7);
1906
1907         /* Thread. */
1908         thread_ip_inc(p);
1909 }
1910
1911 static inline void
1912 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1913 {
1914         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
1915               p->thread_id);
1916
1917         __instr_hdr_extract_exec(p, 8);
1918
1919         /* Thread. */
1920         thread_ip_inc(p);
1921 }
1922
1923 static inline void
1924 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1925 {
1926         struct thread *t = &p->threads[p->thread_id];
1927         struct instruction *ip = t->ip;
1928
1929         uint64_t valid_headers = t->valid_headers;
1930         uint8_t *ptr = t->ptr;
1931         uint32_t offset = t->pkt.offset;
1932         uint32_t length = t->pkt.length;
1933
1934         uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1935         uint32_t header_id = ip->io.hdr.header_id[0];
1936         uint32_t struct_id = ip->io.hdr.struct_id[0];
1937         uint32_t n_bytes = ip->io.hdr.n_bytes[0];
1938
1939         struct header_runtime *h = &t->headers[header_id];
1940
1941         TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
1942               p->thread_id,
1943               header_id,
1944               n_bytes,
1945               n_bytes_last);
1946
1947         n_bytes += n_bytes_last;
1948
1949         /* Headers. */
1950         t->structs[struct_id] = ptr;
1951         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1952         h->n_bytes = n_bytes;
1953
1954         /* Packet. */
1955         t->pkt.offset = offset + n_bytes;
1956         t->pkt.length = length - n_bytes;
1957         t->ptr = ptr + n_bytes;
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 static inline void
1964 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1965 {
1966         struct thread *t = &p->threads[p->thread_id];
1967         struct instruction *ip = t->ip;
1968
1969         uint64_t valid_headers = t->valid_headers;
1970         uint8_t *ptr = t->ptr;
1971
1972         uint32_t header_id = ip->io.hdr.header_id[0];
1973         uint32_t struct_id = ip->io.hdr.struct_id[0];
1974
1975         TRACE("[Thread %2u]: lookahead header %u\n",
1976               p->thread_id,
1977               header_id);
1978
1979         /* Headers. */
1980         t->structs[struct_id] = ptr;
1981         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1982
1983         /* Thread. */
1984         thread_ip_inc(p);
1985 }
1986
1987 /*
1988  * emit.
1989  */
1990 static int
1991 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1992                          struct action *action __rte_unused,
1993                          char **tokens,
1994                          int n_tokens,
1995                          struct instruction *instr,
1996                          struct instruction_data *data __rte_unused)
1997 {
1998         struct header *h;
1999
2000         CHECK(n_tokens == 2, EINVAL);
2001
2002         h = header_parse(p, tokens[1]);
2003         CHECK(h, EINVAL);
2004
2005         instr->type = INSTR_HDR_EMIT;
2006         instr->io.hdr.header_id[0] = h->id;
2007         instr->io.hdr.struct_id[0] = h->struct_id;
2008         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2009         return 0;
2010 }
2011
2012 static inline void
2013 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2014
2015 static inline void
2016 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2017 {
2018         struct thread *t = &p->threads[p->thread_id];
2019         struct instruction *ip = t->ip;
2020         uint64_t valid_headers = t->valid_headers;
2021         uint32_t n_headers_out = t->n_headers_out;
2022         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2023         uint8_t *ho_ptr = NULL;
2024         uint32_t ho_nbytes = 0, first = 1, i;
2025
2026         for (i = 0; i < n_emit; i++) {
2027                 uint32_t header_id = ip->io.hdr.header_id[i];
2028                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2029
2030                 struct header_runtime *hi = &t->headers[header_id];
2031                 uint8_t *hi_ptr0 = hi->ptr0;
2032                 uint32_t n_bytes = hi->n_bytes;
2033
2034                 uint8_t *hi_ptr = t->structs[struct_id];
2035
2036                 if (!MASK64_BIT_GET(valid_headers, header_id))
2037                         continue;
2038
2039                 TRACE("[Thread %2u]: emit header %u\n",
2040                       p->thread_id,
2041                       header_id);
2042
2043                 /* Headers. */
2044                 if (first) {
2045                         first = 0;
2046
2047                         if (!t->n_headers_out) {
2048                                 ho = &t->headers_out[0];
2049
2050                                 ho->ptr0 = hi_ptr0;
2051                                 ho->ptr = hi_ptr;
2052
2053                                 ho_ptr = hi_ptr;
2054                                 ho_nbytes = n_bytes;
2055
2056                                 n_headers_out = 1;
2057
2058                                 continue;
2059                         } else {
2060                                 ho_ptr = ho->ptr;
2061                                 ho_nbytes = ho->n_bytes;
2062                         }
2063                 }
2064
2065                 if (ho_ptr + ho_nbytes == hi_ptr) {
2066                         ho_nbytes += n_bytes;
2067                 } else {
2068                         ho->n_bytes = ho_nbytes;
2069
2070                         ho++;
2071                         ho->ptr0 = hi_ptr0;
2072                         ho->ptr = hi_ptr;
2073
2074                         ho_ptr = hi_ptr;
2075                         ho_nbytes = n_bytes;
2076
2077                         n_headers_out++;
2078                 }
2079         }
2080
2081         ho->n_bytes = ho_nbytes;
2082         t->n_headers_out = n_headers_out;
2083 }
2084
2085 static inline void
2086 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2087 {
2088         __instr_hdr_emit_exec(p, 1);
2089
2090         /* Thread. */
2091         thread_ip_inc(p);
2092 }
2093
2094 static inline void
2095 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2096 {
2097         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2098               p->thread_id);
2099
2100         __instr_hdr_emit_exec(p, 1);
2101         instr_tx_exec(p);
2102 }
2103
2104 static inline void
2105 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2106 {
2107         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2108               p->thread_id);
2109
2110         __instr_hdr_emit_exec(p, 2);
2111         instr_tx_exec(p);
2112 }
2113
2114 static inline void
2115 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2116 {
2117         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2118               p->thread_id);
2119
2120         __instr_hdr_emit_exec(p, 3);
2121         instr_tx_exec(p);
2122 }
2123
2124 static inline void
2125 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2126 {
2127         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2128               p->thread_id);
2129
2130         __instr_hdr_emit_exec(p, 4);
2131         instr_tx_exec(p);
2132 }
2133
2134 static inline void
2135 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2136 {
2137         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2138               p->thread_id);
2139
2140         __instr_hdr_emit_exec(p, 5);
2141         instr_tx_exec(p);
2142 }
2143
2144 static inline void
2145 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2146 {
2147         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2148               p->thread_id);
2149
2150         __instr_hdr_emit_exec(p, 6);
2151         instr_tx_exec(p);
2152 }
2153
2154 static inline void
2155 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2156 {
2157         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2158               p->thread_id);
2159
2160         __instr_hdr_emit_exec(p, 7);
2161         instr_tx_exec(p);
2162 }
2163
2164 static inline void
2165 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2166 {
2167         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2168               p->thread_id);
2169
2170         __instr_hdr_emit_exec(p, 8);
2171         instr_tx_exec(p);
2172 }
2173
2174 /*
2175  * validate.
2176  */
2177 static int
2178 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2179                              struct action *action __rte_unused,
2180                              char **tokens,
2181                              int n_tokens,
2182                              struct instruction *instr,
2183                              struct instruction_data *data __rte_unused)
2184 {
2185         struct header *h;
2186
2187         CHECK(n_tokens == 2, EINVAL);
2188
2189         h = header_parse(p, tokens[1]);
2190         CHECK(h, EINVAL);
2191
2192         instr->type = INSTR_HDR_VALIDATE;
2193         instr->valid.header_id = h->id;
2194         return 0;
2195 }
2196
2197 static inline void
2198 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2199 {
2200         struct thread *t = &p->threads[p->thread_id];
2201         struct instruction *ip = t->ip;
2202         uint32_t header_id = ip->valid.header_id;
2203
2204         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2205
2206         /* Headers. */
2207         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2208
2209         /* Thread. */
2210         thread_ip_inc(p);
2211 }
2212
2213 /*
2214  * invalidate.
2215  */
2216 static int
2217 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2218                                struct action *action __rte_unused,
2219                                char **tokens,
2220                                int n_tokens,
2221                                struct instruction *instr,
2222                                struct instruction_data *data __rte_unused)
2223 {
2224         struct header *h;
2225
2226         CHECK(n_tokens == 2, EINVAL);
2227
2228         h = header_parse(p, tokens[1]);
2229         CHECK(h, EINVAL);
2230
2231         instr->type = INSTR_HDR_INVALIDATE;
2232         instr->valid.header_id = h->id;
2233         return 0;
2234 }
2235
2236 static inline void
2237 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2238 {
2239         struct thread *t = &p->threads[p->thread_id];
2240         struct instruction *ip = t->ip;
2241         uint32_t header_id = ip->valid.header_id;
2242
2243         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2244
2245         /* Headers. */
2246         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2247
2248         /* Thread. */
2249         thread_ip_inc(p);
2250 }
2251
2252 /*
2253  * table.
2254  */
2255 static struct table *
2256 table_find(struct rte_swx_pipeline *p, const char *name);
2257
2258 static struct selector *
2259 selector_find(struct rte_swx_pipeline *p, const char *name);
2260
2261 static struct learner *
2262 learner_find(struct rte_swx_pipeline *p, const char *name);
2263
2264 static int
2265 instr_table_translate(struct rte_swx_pipeline *p,
2266                       struct action *action,
2267                       char **tokens,
2268                       int n_tokens,
2269                       struct instruction *instr,
2270                       struct instruction_data *data __rte_unused)
2271 {
2272         struct table *t;
2273         struct selector *s;
2274         struct learner *l;
2275
2276         CHECK(!action, EINVAL);
2277         CHECK(n_tokens == 2, EINVAL);
2278
2279         t = table_find(p, tokens[1]);
2280         if (t) {
2281                 instr->type = INSTR_TABLE;
2282                 instr->table.table_id = t->id;
2283                 return 0;
2284         }
2285
2286         s = selector_find(p, tokens[1]);
2287         if (s) {
2288                 instr->type = INSTR_SELECTOR;
2289                 instr->table.table_id = s->id;
2290                 return 0;
2291         }
2292
2293         l = learner_find(p, tokens[1]);
2294         if (l) {
2295                 instr->type = INSTR_LEARNER;
2296                 instr->table.table_id = l->id;
2297                 return 0;
2298         }
2299
2300         CHECK(0, EINVAL);
2301 }
2302
2303 static inline void
2304 instr_table_exec(struct rte_swx_pipeline *p)
2305 {
2306         struct thread *t = &p->threads[p->thread_id];
2307         struct instruction *ip = t->ip;
2308         uint32_t table_id = ip->table.table_id;
2309         struct rte_swx_table_state *ts = &t->table_state[table_id];
2310         struct table_runtime *table = &t->tables[table_id];
2311         struct table_statistics *stats = &p->table_stats[table_id];
2312         uint64_t action_id, n_pkts_hit, n_pkts_action;
2313         uint8_t *action_data;
2314         int done, hit;
2315
2316         /* Table. */
2317         done = table->func(ts->obj,
2318                            table->mailbox,
2319                            table->key,
2320                            &action_id,
2321                            &action_data,
2322                            &hit);
2323         if (!done) {
2324                 /* Thread. */
2325                 TRACE("[Thread %2u] table %u (not finalized)\n",
2326                       p->thread_id,
2327                       table_id);
2328
2329                 thread_yield(p);
2330                 return;
2331         }
2332
2333         action_id = hit ? action_id : ts->default_action_id;
2334         action_data = hit ? action_data : ts->default_action_data;
2335         n_pkts_hit = stats->n_pkts_hit[hit];
2336         n_pkts_action = stats->n_pkts_action[action_id];
2337
2338         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2339               p->thread_id,
2340               table_id,
2341               hit ? "hit" : "miss",
2342               (uint32_t)action_id);
2343
2344         t->action_id = action_id;
2345         t->structs[0] = action_data;
2346         t->hit = hit;
2347         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2348         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2349
2350         /* Thread. */
2351         thread_ip_action_call(p, t, action_id);
2352 }
2353
2354 static inline void
2355 instr_selector_exec(struct rte_swx_pipeline *p)
2356 {
2357         struct thread *t = &p->threads[p->thread_id];
2358         struct instruction *ip = t->ip;
2359         uint32_t selector_id = ip->table.table_id;
2360         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2361         struct selector_runtime *selector = &t->selectors[selector_id];
2362         struct selector_statistics *stats = &p->selector_stats[selector_id];
2363         uint64_t n_pkts = stats->n_pkts;
2364         int done;
2365
2366         /* Table. */
2367         done = rte_swx_table_selector_select(ts->obj,
2368                            selector->mailbox,
2369                            selector->group_id_buffer,
2370                            selector->selector_buffer,
2371                            selector->member_id_buffer);
2372         if (!done) {
2373                 /* Thread. */
2374                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2375                       p->thread_id,
2376                       selector_id);
2377
2378                 thread_yield(p);
2379                 return;
2380         }
2381
2382
2383         TRACE("[Thread %2u] selector %u\n",
2384               p->thread_id,
2385               selector_id);
2386
2387         stats->n_pkts = n_pkts + 1;
2388
2389         /* Thread. */
2390         thread_ip_inc(p);
2391 }
2392
2393 static inline void
2394 instr_learner_exec(struct rte_swx_pipeline *p)
2395 {
2396         struct thread *t = &p->threads[p->thread_id];
2397         struct instruction *ip = t->ip;
2398         uint32_t learner_id = ip->table.table_id;
2399         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2400                 p->n_selectors + learner_id];
2401         struct learner_runtime *l = &t->learners[learner_id];
2402         struct learner_statistics *stats = &p->learner_stats[learner_id];
2403         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2404         uint8_t *action_data;
2405         int done, hit;
2406
2407         /* Table. */
2408         time = rte_get_tsc_cycles();
2409
2410         done = rte_swx_table_learner_lookup(ts->obj,
2411                                             l->mailbox,
2412                                             time,
2413                                             l->key,
2414                                             &action_id,
2415                                             &action_data,
2416                                             &hit);
2417         if (!done) {
2418                 /* Thread. */
2419                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2420                       p->thread_id,
2421                       learner_id);
2422
2423                 thread_yield(p);
2424                 return;
2425         }
2426
2427         action_id = hit ? action_id : ts->default_action_id;
2428         action_data = hit ? action_data : ts->default_action_data;
2429         n_pkts_hit = stats->n_pkts_hit[hit];
2430         n_pkts_action = stats->n_pkts_action[action_id];
2431
2432         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2433               p->thread_id,
2434               learner_id,
2435               hit ? "hit" : "miss",
2436               (uint32_t)action_id);
2437
2438         t->action_id = action_id;
2439         t->structs[0] = action_data;
2440         t->hit = hit;
2441         t->learner_id = learner_id;
2442         t->time = time;
2443         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2444         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2445
2446         /* Thread. */
2447         thread_ip_action_call(p, t, action_id);
2448 }
2449
2450 /*
2451  * learn.
2452  */
2453 static struct action *
2454 action_find(struct rte_swx_pipeline *p, const char *name);
2455
2456 static int
2457 action_has_nbo_args(struct action *a);
2458
2459 static int
2460 instr_learn_translate(struct rte_swx_pipeline *p,
2461                       struct action *action,
2462                       char **tokens,
2463                       int n_tokens,
2464                       struct instruction *instr,
2465                       struct instruction_data *data __rte_unused)
2466 {
2467         struct action *a;
2468
2469         CHECK(action, EINVAL);
2470         CHECK(n_tokens == 2, EINVAL);
2471
2472         a = action_find(p, tokens[1]);
2473         CHECK(a, EINVAL);
2474         CHECK(!action_has_nbo_args(a), EINVAL);
2475
2476         instr->type = INSTR_LEARNER_LEARN;
2477         instr->learn.action_id = a->id;
2478
2479         return 0;
2480 }
2481
2482 static inline void
2483 instr_learn_exec(struct rte_swx_pipeline *p)
2484 {
2485         struct thread *t = &p->threads[p->thread_id];
2486         struct instruction *ip = t->ip;
2487         uint64_t action_id = ip->learn.action_id;
2488         uint32_t learner_id = t->learner_id;
2489         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2490                 p->n_selectors + learner_id];
2491         struct learner_runtime *l = &t->learners[learner_id];
2492         struct learner_statistics *stats = &p->learner_stats[learner_id];
2493         uint32_t status;
2494
2495         /* Table. */
2496         status = rte_swx_table_learner_add(ts->obj,
2497                                            l->mailbox,
2498                                            t->time,
2499                                            action_id,
2500                                            l->action_data[action_id]);
2501
2502         TRACE("[Thread %2u] learner %u learn %s\n",
2503               p->thread_id,
2504               learner_id,
2505               status ? "ok" : "error");
2506
2507         stats->n_pkts_learn[status] += 1;
2508
2509         /* Thread. */
2510         thread_ip_inc(p);
2511 }
2512
2513 /*
2514  * forget.
2515  */
2516 static int
2517 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2518                        struct action *action,
2519                        char **tokens __rte_unused,
2520                        int n_tokens,
2521                        struct instruction *instr,
2522                        struct instruction_data *data __rte_unused)
2523 {
2524         CHECK(action, EINVAL);
2525         CHECK(n_tokens == 1, EINVAL);
2526
2527         instr->type = INSTR_LEARNER_FORGET;
2528
2529         return 0;
2530 }
2531
2532 static inline void
2533 instr_forget_exec(struct rte_swx_pipeline *p)
2534 {
2535         struct thread *t = &p->threads[p->thread_id];
2536         uint32_t learner_id = t->learner_id;
2537         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2538                 p->n_selectors + learner_id];
2539         struct learner_runtime *l = &t->learners[learner_id];
2540         struct learner_statistics *stats = &p->learner_stats[learner_id];
2541
2542         /* Table. */
2543         rte_swx_table_learner_delete(ts->obj, l->mailbox);
2544
2545         TRACE("[Thread %2u] learner %u forget\n",
2546               p->thread_id,
2547               learner_id);
2548
2549         stats->n_pkts_forget += 1;
2550
2551         /* Thread. */
2552         thread_ip_inc(p);
2553 }
2554
2555 /*
2556  * extern.
2557  */
2558 static int
2559 instr_extern_translate(struct rte_swx_pipeline *p,
2560                        struct action *action __rte_unused,
2561                        char **tokens,
2562                        int n_tokens,
2563                        struct instruction *instr,
2564                        struct instruction_data *data __rte_unused)
2565 {
2566         char *token = tokens[1];
2567
2568         CHECK(n_tokens == 2, EINVAL);
2569
2570         if (token[0] == 'e') {
2571                 struct extern_obj *obj;
2572                 struct extern_type_member_func *func;
2573
2574                 func = extern_obj_member_func_parse(p, token, &obj);
2575                 CHECK(func, EINVAL);
2576
2577                 instr->type = INSTR_EXTERN_OBJ;
2578                 instr->ext_obj.ext_obj_id = obj->id;
2579                 instr->ext_obj.func_id = func->id;
2580
2581                 return 0;
2582         }
2583
2584         if (token[0] == 'f') {
2585                 struct extern_func *func;
2586
2587                 func = extern_func_parse(p, token);
2588                 CHECK(func, EINVAL);
2589
2590                 instr->type = INSTR_EXTERN_FUNC;
2591                 instr->ext_func.ext_func_id = func->id;
2592
2593                 return 0;
2594         }
2595
2596         CHECK(0, EINVAL);
2597 }
2598
2599 static inline void
2600 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2601 {
2602         struct thread *t = &p->threads[p->thread_id];
2603         struct instruction *ip = t->ip;
2604         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2605         uint32_t func_id = ip->ext_obj.func_id;
2606         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2607         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2608
2609         TRACE("[Thread %2u] extern obj %u member func %u\n",
2610               p->thread_id,
2611               obj_id,
2612               func_id);
2613
2614         /* Extern object member function execute. */
2615         uint32_t done = func(obj->obj, obj->mailbox);
2616
2617         /* Thread. */
2618         thread_ip_inc_cond(t, done);
2619         thread_yield_cond(p, done ^ 1);
2620 }
2621
2622 static inline void
2623 instr_extern_func_exec(struct rte_swx_pipeline *p)
2624 {
2625         struct thread *t = &p->threads[p->thread_id];
2626         struct instruction *ip = t->ip;
2627         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2628         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2629         rte_swx_extern_func_t func = ext_func->func;
2630
2631         TRACE("[Thread %2u] extern func %u\n",
2632               p->thread_id,
2633               ext_func_id);
2634
2635         /* Extern function execute. */
2636         uint32_t done = func(ext_func->mailbox);
2637
2638         /* Thread. */
2639         thread_ip_inc_cond(t, done);
2640         thread_yield_cond(p, done ^ 1);
2641 }
2642
2643 /*
2644  * mov.
2645  */
2646 static int
2647 instr_mov_translate(struct rte_swx_pipeline *p,
2648                     struct action *action,
2649                     char **tokens,
2650                     int n_tokens,
2651                     struct instruction *instr,
2652                     struct instruction_data *data __rte_unused)
2653 {
2654         char *dst = tokens[1], *src = tokens[2];
2655         struct field *fdst, *fsrc;
2656         uint64_t src_val;
2657         uint32_t dst_struct_id = 0, src_struct_id = 0;
2658
2659         CHECK(n_tokens == 3, EINVAL);
2660
2661         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2662         CHECK(fdst, EINVAL);
2663         CHECK(!fdst->var_size, EINVAL);
2664
2665         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2666         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2667         if (fsrc) {
2668                 CHECK(!fsrc->var_size, EINVAL);
2669
2670                 instr->type = INSTR_MOV;
2671                 if (dst[0] != 'h' && src[0] == 'h')
2672                         instr->type = INSTR_MOV_MH;
2673                 if (dst[0] == 'h' && src[0] != 'h')
2674                         instr->type = INSTR_MOV_HM;
2675                 if (dst[0] == 'h' && src[0] == 'h')
2676                         instr->type = INSTR_MOV_HH;
2677
2678                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2679                 instr->mov.dst.n_bits = fdst->n_bits;
2680                 instr->mov.dst.offset = fdst->offset / 8;
2681                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2682                 instr->mov.src.n_bits = fsrc->n_bits;
2683                 instr->mov.src.offset = fsrc->offset / 8;
2684                 return 0;
2685         }
2686
2687         /* MOV_I. */
2688         src_val = strtoull(src, &src, 0);
2689         CHECK(!src[0], EINVAL);
2690
2691         if (dst[0] == 'h')
2692                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2693
2694         instr->type = INSTR_MOV_I;
2695         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2696         instr->mov.dst.n_bits = fdst->n_bits;
2697         instr->mov.dst.offset = fdst->offset / 8;
2698         instr->mov.src_val = src_val;
2699         return 0;
2700 }
2701
2702 static inline void
2703 instr_mov_exec(struct rte_swx_pipeline *p)
2704 {
2705         struct thread *t = &p->threads[p->thread_id];
2706         struct instruction *ip = t->ip;
2707
2708         TRACE("[Thread %2u] mov\n",
2709               p->thread_id);
2710
2711         MOV(t, ip);
2712
2713         /* Thread. */
2714         thread_ip_inc(p);
2715 }
2716
2717 static inline void
2718 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2719 {
2720         struct thread *t = &p->threads[p->thread_id];
2721         struct instruction *ip = t->ip;
2722
2723         TRACE("[Thread %2u] mov (mh)\n",
2724               p->thread_id);
2725
2726         MOV_MH(t, ip);
2727
2728         /* Thread. */
2729         thread_ip_inc(p);
2730 }
2731
2732 static inline void
2733 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2734 {
2735         struct thread *t = &p->threads[p->thread_id];
2736         struct instruction *ip = t->ip;
2737
2738         TRACE("[Thread %2u] mov (hm)\n",
2739               p->thread_id);
2740
2741         MOV_HM(t, ip);
2742
2743         /* Thread. */
2744         thread_ip_inc(p);
2745 }
2746
2747 static inline void
2748 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2749 {
2750         struct thread *t = &p->threads[p->thread_id];
2751         struct instruction *ip = t->ip;
2752
2753         TRACE("[Thread %2u] mov (hh)\n",
2754               p->thread_id);
2755
2756         MOV_HH(t, ip);
2757
2758         /* Thread. */
2759         thread_ip_inc(p);
2760 }
2761
2762 static inline void
2763 instr_mov_i_exec(struct rte_swx_pipeline *p)
2764 {
2765         struct thread *t = &p->threads[p->thread_id];
2766         struct instruction *ip = t->ip;
2767
2768         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2769               p->thread_id,
2770               ip->mov.src_val);
2771
2772         MOV_I(t, ip);
2773
2774         /* Thread. */
2775         thread_ip_inc(p);
2776 }
2777
2778 /*
2779  * dma.
2780  */
2781 static inline void
2782 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2783
2784 static inline void
2785 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2786 {
2787         struct thread *t = &p->threads[p->thread_id];
2788         struct instruction *ip = t->ip;
2789         uint8_t *action_data = t->structs[0];
2790         uint64_t valid_headers = t->valid_headers;
2791         uint32_t i;
2792
2793         for (i = 0; i < n_dma; i++) {
2794                 uint32_t header_id = ip->dma.dst.header_id[i];
2795                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2796                 uint32_t offset = ip->dma.src.offset[i];
2797                 uint32_t n_bytes = ip->dma.n_bytes[i];
2798
2799                 struct header_runtime *h = &t->headers[header_id];
2800                 uint8_t *h_ptr0 = h->ptr0;
2801                 uint8_t *h_ptr = t->structs[struct_id];
2802
2803                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2804                         h_ptr : h_ptr0;
2805                 void *src = &action_data[offset];
2806
2807                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2808
2809                 /* Headers. */
2810                 memcpy(dst, src, n_bytes);
2811                 t->structs[struct_id] = dst;
2812                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2813         }
2814
2815         t->valid_headers = valid_headers;
2816 }
2817
2818 static inline void
2819 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2820 {
2821         __instr_dma_ht_exec(p, 1);
2822
2823         /* Thread. */
2824         thread_ip_inc(p);
2825 }
2826
2827 static inline void
2828 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2829 {
2830         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2831               p->thread_id);
2832
2833         __instr_dma_ht_exec(p, 2);
2834
2835         /* Thread. */
2836         thread_ip_inc(p);
2837 }
2838
2839 static inline void
2840 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2841 {
2842         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2843               p->thread_id);
2844
2845         __instr_dma_ht_exec(p, 3);
2846
2847         /* Thread. */
2848         thread_ip_inc(p);
2849 }
2850
2851 static inline void
2852 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2853 {
2854         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2855               p->thread_id);
2856
2857         __instr_dma_ht_exec(p, 4);
2858
2859         /* Thread. */
2860         thread_ip_inc(p);
2861 }
2862
2863 static inline void
2864 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2865 {
2866         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2867               p->thread_id);
2868
2869         __instr_dma_ht_exec(p, 5);
2870
2871         /* Thread. */
2872         thread_ip_inc(p);
2873 }
2874
2875 static inline void
2876 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2877 {
2878         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2879               p->thread_id);
2880
2881         __instr_dma_ht_exec(p, 6);
2882
2883         /* Thread. */
2884         thread_ip_inc(p);
2885 }
2886
2887 static inline void
2888 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2889 {
2890         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2891               p->thread_id);
2892
2893         __instr_dma_ht_exec(p, 7);
2894
2895         /* Thread. */
2896         thread_ip_inc(p);
2897 }
2898
2899 static inline void
2900 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2901 {
2902         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2903               p->thread_id);
2904
2905         __instr_dma_ht_exec(p, 8);
2906
2907         /* Thread. */
2908         thread_ip_inc(p);
2909 }
2910
2911 /*
2912  * alu.
2913  */
2914 static int
2915 instr_alu_add_translate(struct rte_swx_pipeline *p,
2916                         struct action *action,
2917                         char **tokens,
2918                         int n_tokens,
2919                         struct instruction *instr,
2920                         struct instruction_data *data __rte_unused)
2921 {
2922         char *dst = tokens[1], *src = tokens[2];
2923         struct field *fdst, *fsrc;
2924         uint64_t src_val;
2925         uint32_t dst_struct_id = 0, src_struct_id = 0;
2926
2927         CHECK(n_tokens == 3, EINVAL);
2928
2929         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2930         CHECK(fdst, EINVAL);
2931         CHECK(!fdst->var_size, EINVAL);
2932
2933         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2934         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2935         if (fsrc) {
2936                 CHECK(!fsrc->var_size, EINVAL);
2937
2938                 instr->type = INSTR_ALU_ADD;
2939                 if (dst[0] == 'h' && src[0] != 'h')
2940                         instr->type = INSTR_ALU_ADD_HM;
2941                 if (dst[0] != 'h' && src[0] == 'h')
2942                         instr->type = INSTR_ALU_ADD_MH;
2943                 if (dst[0] == 'h' && src[0] == 'h')
2944                         instr->type = INSTR_ALU_ADD_HH;
2945
2946                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2947                 instr->alu.dst.n_bits = fdst->n_bits;
2948                 instr->alu.dst.offset = fdst->offset / 8;
2949                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2950                 instr->alu.src.n_bits = fsrc->n_bits;
2951                 instr->alu.src.offset = fsrc->offset / 8;
2952                 return 0;
2953         }
2954
2955         /* ADD_MI, ADD_HI. */
2956         src_val = strtoull(src, &src, 0);
2957         CHECK(!src[0], EINVAL);
2958
2959         instr->type = INSTR_ALU_ADD_MI;
2960         if (dst[0] == 'h')
2961                 instr->type = INSTR_ALU_ADD_HI;
2962
2963         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2964         instr->alu.dst.n_bits = fdst->n_bits;
2965         instr->alu.dst.offset = fdst->offset / 8;
2966         instr->alu.src_val = src_val;
2967         return 0;
2968 }
2969
2970 static int
2971 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2972                         struct action *action,
2973                         char **tokens,
2974                         int n_tokens,
2975                         struct instruction *instr,
2976                         struct instruction_data *data __rte_unused)
2977 {
2978         char *dst = tokens[1], *src = tokens[2];
2979         struct field *fdst, *fsrc;
2980         uint64_t src_val;
2981         uint32_t dst_struct_id = 0, src_struct_id = 0;
2982
2983         CHECK(n_tokens == 3, EINVAL);
2984
2985         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2986         CHECK(fdst, EINVAL);
2987         CHECK(!fdst->var_size, EINVAL);
2988
2989         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2990         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2991         if (fsrc) {
2992                 CHECK(!fsrc->var_size, EINVAL);
2993
2994                 instr->type = INSTR_ALU_SUB;
2995                 if (dst[0] == 'h' && src[0] != 'h')
2996                         instr->type = INSTR_ALU_SUB_HM;
2997                 if (dst[0] != 'h' && src[0] == 'h')
2998                         instr->type = INSTR_ALU_SUB_MH;
2999                 if (dst[0] == 'h' && src[0] == 'h')
3000                         instr->type = INSTR_ALU_SUB_HH;
3001
3002                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3003                 instr->alu.dst.n_bits = fdst->n_bits;
3004                 instr->alu.dst.offset = fdst->offset / 8;
3005                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3006                 instr->alu.src.n_bits = fsrc->n_bits;
3007                 instr->alu.src.offset = fsrc->offset / 8;
3008                 return 0;
3009         }
3010
3011         /* SUB_MI, SUB_HI. */
3012         src_val = strtoull(src, &src, 0);
3013         CHECK(!src[0], EINVAL);
3014
3015         instr->type = INSTR_ALU_SUB_MI;
3016         if (dst[0] == 'h')
3017                 instr->type = INSTR_ALU_SUB_HI;
3018
3019         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3020         instr->alu.dst.n_bits = fdst->n_bits;
3021         instr->alu.dst.offset = fdst->offset / 8;
3022         instr->alu.src_val = src_val;
3023         return 0;
3024 }
3025
3026 static int
3027 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3028                           struct action *action __rte_unused,
3029                           char **tokens,
3030                           int n_tokens,
3031                           struct instruction *instr,
3032                           struct instruction_data *data __rte_unused)
3033 {
3034         char *dst = tokens[1], *src = tokens[2];
3035         struct header *hdst, *hsrc;
3036         struct field *fdst, *fsrc;
3037
3038         CHECK(n_tokens == 3, EINVAL);
3039
3040         fdst = header_field_parse(p, dst, &hdst);
3041         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3042         CHECK(!fdst->var_size, EINVAL);
3043
3044         /* CKADD_FIELD. */
3045         fsrc = header_field_parse(p, src, &hsrc);
3046         if (fsrc) {
3047                 CHECK(!fsrc->var_size, EINVAL);
3048
3049                 instr->type = INSTR_ALU_CKADD_FIELD;
3050                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3051                 instr->alu.dst.n_bits = fdst->n_bits;
3052                 instr->alu.dst.offset = fdst->offset / 8;
3053                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3054                 instr->alu.src.n_bits = fsrc->n_bits;
3055                 instr->alu.src.offset = fsrc->offset / 8;
3056                 return 0;
3057         }
3058
3059         /* CKADD_STRUCT, CKADD_STRUCT20. */
3060         hsrc = header_parse(p, src);
3061         CHECK(hsrc, EINVAL);
3062         CHECK(!hsrc->st->var_size, EINVAL);
3063
3064         instr->type = INSTR_ALU_CKADD_STRUCT;
3065         if ((hsrc->st->n_bits / 8) == 20)
3066                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3067
3068         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3069         instr->alu.dst.n_bits = fdst->n_bits;
3070         instr->alu.dst.offset = fdst->offset / 8;
3071         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3072         instr->alu.src.n_bits = hsrc->st->n_bits;
3073         instr->alu.src.offset = 0; /* Unused. */
3074         return 0;
3075 }
3076
3077 static int
3078 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3079                           struct action *action __rte_unused,
3080                           char **tokens,
3081                           int n_tokens,
3082                           struct instruction *instr,
3083                           struct instruction_data *data __rte_unused)
3084 {
3085         char *dst = tokens[1], *src = tokens[2];
3086         struct header *hdst, *hsrc;
3087         struct field *fdst, *fsrc;
3088
3089         CHECK(n_tokens == 3, EINVAL);
3090
3091         fdst = header_field_parse(p, dst, &hdst);
3092         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3093         CHECK(!fdst->var_size, EINVAL);
3094
3095         fsrc = header_field_parse(p, src, &hsrc);
3096         CHECK(fsrc, EINVAL);
3097         CHECK(!fsrc->var_size, EINVAL);
3098
3099         instr->type = INSTR_ALU_CKSUB_FIELD;
3100         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3101         instr->alu.dst.n_bits = fdst->n_bits;
3102         instr->alu.dst.offset = fdst->offset / 8;
3103         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3104         instr->alu.src.n_bits = fsrc->n_bits;
3105         instr->alu.src.offset = fsrc->offset / 8;
3106         return 0;
3107 }
3108
3109 static int
3110 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3111                         struct action *action,
3112                         char **tokens,
3113                         int n_tokens,
3114                         struct instruction *instr,
3115                         struct instruction_data *data __rte_unused)
3116 {
3117         char *dst = tokens[1], *src = tokens[2];
3118         struct field *fdst, *fsrc;
3119         uint64_t src_val;
3120         uint32_t dst_struct_id = 0, src_struct_id = 0;
3121
3122         CHECK(n_tokens == 3, EINVAL);
3123
3124         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3125         CHECK(fdst, EINVAL);
3126         CHECK(!fdst->var_size, EINVAL);
3127
3128         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3129         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3130         if (fsrc) {
3131                 CHECK(!fsrc->var_size, EINVAL);
3132
3133                 instr->type = INSTR_ALU_SHL;
3134                 if (dst[0] == 'h' && src[0] != 'h')
3135                         instr->type = INSTR_ALU_SHL_HM;
3136                 if (dst[0] != 'h' && src[0] == 'h')
3137                         instr->type = INSTR_ALU_SHL_MH;
3138                 if (dst[0] == 'h' && src[0] == 'h')
3139                         instr->type = INSTR_ALU_SHL_HH;
3140
3141                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3142                 instr->alu.dst.n_bits = fdst->n_bits;
3143                 instr->alu.dst.offset = fdst->offset / 8;
3144                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3145                 instr->alu.src.n_bits = fsrc->n_bits;
3146                 instr->alu.src.offset = fsrc->offset / 8;
3147                 return 0;
3148         }
3149
3150         /* SHL_MI, SHL_HI. */
3151         src_val = strtoull(src, &src, 0);
3152         CHECK(!src[0], EINVAL);
3153
3154         instr->type = INSTR_ALU_SHL_MI;
3155         if (dst[0] == 'h')
3156                 instr->type = INSTR_ALU_SHL_HI;
3157
3158         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3159         instr->alu.dst.n_bits = fdst->n_bits;
3160         instr->alu.dst.offset = fdst->offset / 8;
3161         instr->alu.src_val = src_val;
3162         return 0;
3163 }
3164
3165 static int
3166 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3167                         struct action *action,
3168                         char **tokens,
3169                         int n_tokens,
3170                         struct instruction *instr,
3171                         struct instruction_data *data __rte_unused)
3172 {
3173         char *dst = tokens[1], *src = tokens[2];
3174         struct field *fdst, *fsrc;
3175         uint64_t src_val;
3176         uint32_t dst_struct_id = 0, src_struct_id = 0;
3177
3178         CHECK(n_tokens == 3, EINVAL);
3179
3180         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3181         CHECK(fdst, EINVAL);
3182         CHECK(!fdst->var_size, EINVAL);
3183
3184         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3185         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3186         if (fsrc) {
3187                 CHECK(!fsrc->var_size, EINVAL);
3188
3189                 instr->type = INSTR_ALU_SHR;
3190                 if (dst[0] == 'h' && src[0] != 'h')
3191                         instr->type = INSTR_ALU_SHR_HM;
3192                 if (dst[0] != 'h' && src[0] == 'h')
3193                         instr->type = INSTR_ALU_SHR_MH;
3194                 if (dst[0] == 'h' && src[0] == 'h')
3195                         instr->type = INSTR_ALU_SHR_HH;
3196
3197                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3198                 instr->alu.dst.n_bits = fdst->n_bits;
3199                 instr->alu.dst.offset = fdst->offset / 8;
3200                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3201                 instr->alu.src.n_bits = fsrc->n_bits;
3202                 instr->alu.src.offset = fsrc->offset / 8;
3203                 return 0;
3204         }
3205
3206         /* SHR_MI, SHR_HI. */
3207         src_val = strtoull(src, &src, 0);
3208         CHECK(!src[0], EINVAL);
3209
3210         instr->type = INSTR_ALU_SHR_MI;
3211         if (dst[0] == 'h')
3212                 instr->type = INSTR_ALU_SHR_HI;
3213
3214         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3215         instr->alu.dst.n_bits = fdst->n_bits;
3216         instr->alu.dst.offset = fdst->offset / 8;
3217         instr->alu.src_val = src_val;
3218         return 0;
3219 }
3220
3221 static int
3222 instr_alu_and_translate(struct rte_swx_pipeline *p,
3223                         struct action *action,
3224                         char **tokens,
3225                         int n_tokens,
3226                         struct instruction *instr,
3227                         struct instruction_data *data __rte_unused)
3228 {
3229         char *dst = tokens[1], *src = tokens[2];
3230         struct field *fdst, *fsrc;
3231         uint64_t src_val;
3232         uint32_t dst_struct_id = 0, src_struct_id = 0;
3233
3234         CHECK(n_tokens == 3, EINVAL);
3235
3236         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3237         CHECK(fdst, EINVAL);
3238         CHECK(!fdst->var_size, EINVAL);
3239
3240         /* AND, AND_MH, AND_HM, AND_HH. */
3241         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3242         if (fsrc) {
3243                 CHECK(!fsrc->var_size, EINVAL);
3244
3245                 instr->type = INSTR_ALU_AND;
3246                 if (dst[0] != 'h' && src[0] == 'h')
3247                         instr->type = INSTR_ALU_AND_MH;
3248                 if (dst[0] == 'h' && src[0] != 'h')
3249                         instr->type = INSTR_ALU_AND_HM;
3250                 if (dst[0] == 'h' && src[0] == 'h')
3251                         instr->type = INSTR_ALU_AND_HH;
3252
3253                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3254                 instr->alu.dst.n_bits = fdst->n_bits;
3255                 instr->alu.dst.offset = fdst->offset / 8;
3256                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3257                 instr->alu.src.n_bits = fsrc->n_bits;
3258                 instr->alu.src.offset = fsrc->offset / 8;
3259                 return 0;
3260         }
3261
3262         /* AND_I. */
3263         src_val = strtoull(src, &src, 0);
3264         CHECK(!src[0], EINVAL);
3265
3266         if (dst[0] == 'h')
3267                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3268
3269         instr->type = INSTR_ALU_AND_I;
3270         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3271         instr->alu.dst.n_bits = fdst->n_bits;
3272         instr->alu.dst.offset = fdst->offset / 8;
3273         instr->alu.src_val = src_val;
3274         return 0;
3275 }
3276
3277 static int
3278 instr_alu_or_translate(struct rte_swx_pipeline *p,
3279                        struct action *action,
3280                        char **tokens,
3281                        int n_tokens,
3282                        struct instruction *instr,
3283                        struct instruction_data *data __rte_unused)
3284 {
3285         char *dst = tokens[1], *src = tokens[2];
3286         struct field *fdst, *fsrc;
3287         uint64_t src_val;
3288         uint32_t dst_struct_id = 0, src_struct_id = 0;
3289
3290         CHECK(n_tokens == 3, EINVAL);
3291
3292         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3293         CHECK(fdst, EINVAL);
3294         CHECK(!fdst->var_size, EINVAL);
3295
3296         /* OR, OR_MH, OR_HM, OR_HH. */
3297         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3298         if (fsrc) {
3299                 CHECK(!fsrc->var_size, EINVAL);
3300
3301                 instr->type = INSTR_ALU_OR;
3302                 if (dst[0] != 'h' && src[0] == 'h')
3303                         instr->type = INSTR_ALU_OR_MH;
3304                 if (dst[0] == 'h' && src[0] != 'h')
3305                         instr->type = INSTR_ALU_OR_HM;
3306                 if (dst[0] == 'h' && src[0] == 'h')
3307                         instr->type = INSTR_ALU_OR_HH;
3308
3309                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3310                 instr->alu.dst.n_bits = fdst->n_bits;
3311                 instr->alu.dst.offset = fdst->offset / 8;
3312                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3313                 instr->alu.src.n_bits = fsrc->n_bits;
3314                 instr->alu.src.offset = fsrc->offset / 8;
3315                 return 0;
3316         }
3317
3318         /* OR_I. */
3319         src_val = strtoull(src, &src, 0);
3320         CHECK(!src[0], EINVAL);
3321
3322         if (dst[0] == 'h')
3323                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3324
3325         instr->type = INSTR_ALU_OR_I;
3326         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3327         instr->alu.dst.n_bits = fdst->n_bits;
3328         instr->alu.dst.offset = fdst->offset / 8;
3329         instr->alu.src_val = src_val;
3330         return 0;
3331 }
3332
3333 static int
3334 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3335                         struct action *action,
3336                         char **tokens,
3337                         int n_tokens,
3338                         struct instruction *instr,
3339                         struct instruction_data *data __rte_unused)
3340 {
3341         char *dst = tokens[1], *src = tokens[2];
3342         struct field *fdst, *fsrc;
3343         uint64_t src_val;
3344         uint32_t dst_struct_id = 0, src_struct_id = 0;
3345
3346         CHECK(n_tokens == 3, EINVAL);
3347
3348         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3349         CHECK(fdst, EINVAL);
3350         CHECK(!fdst->var_size, EINVAL);
3351
3352         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3353         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3354         if (fsrc) {
3355                 CHECK(!fsrc->var_size, EINVAL);
3356
3357                 instr->type = INSTR_ALU_XOR;
3358                 if (dst[0] != 'h' && src[0] == 'h')
3359                         instr->type = INSTR_ALU_XOR_MH;
3360                 if (dst[0] == 'h' && src[0] != 'h')
3361                         instr->type = INSTR_ALU_XOR_HM;
3362                 if (dst[0] == 'h' && src[0] == 'h')
3363                         instr->type = INSTR_ALU_XOR_HH;
3364
3365                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3366                 instr->alu.dst.n_bits = fdst->n_bits;
3367                 instr->alu.dst.offset = fdst->offset / 8;
3368                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3369                 instr->alu.src.n_bits = fsrc->n_bits;
3370                 instr->alu.src.offset = fsrc->offset / 8;
3371                 return 0;
3372         }
3373
3374         /* XOR_I. */
3375         src_val = strtoull(src, &src, 0);
3376         CHECK(!src[0], EINVAL);
3377
3378         if (dst[0] == 'h')
3379                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3380
3381         instr->type = INSTR_ALU_XOR_I;
3382         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3383         instr->alu.dst.n_bits = fdst->n_bits;
3384         instr->alu.dst.offset = fdst->offset / 8;
3385         instr->alu.src_val = src_val;
3386         return 0;
3387 }
3388
3389 static inline void
3390 instr_alu_add_exec(struct rte_swx_pipeline *p)
3391 {
3392         struct thread *t = &p->threads[p->thread_id];
3393         struct instruction *ip = t->ip;
3394
3395         TRACE("[Thread %2u] add\n", p->thread_id);
3396
3397         /* Structs. */
3398         ALU(t, ip, +);
3399
3400         /* Thread. */
3401         thread_ip_inc(p);
3402 }
3403
3404 static inline void
3405 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3406 {
3407         struct thread *t = &p->threads[p->thread_id];
3408         struct instruction *ip = t->ip;
3409
3410         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3411
3412         /* Structs. */
3413         ALU_MH(t, ip, +);
3414
3415         /* Thread. */
3416         thread_ip_inc(p);
3417 }
3418
3419 static inline void
3420 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3421 {
3422         struct thread *t = &p->threads[p->thread_id];
3423         struct instruction *ip = t->ip;
3424
3425         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3426
3427         /* Structs. */
3428         ALU_HM(t, ip, +);
3429
3430         /* Thread. */
3431         thread_ip_inc(p);
3432 }
3433
3434 static inline void
3435 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3436 {
3437         struct thread *t = &p->threads[p->thread_id];
3438         struct instruction *ip = t->ip;
3439
3440         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3441
3442         /* Structs. */
3443         ALU_HH(t, ip, +);
3444
3445         /* Thread. */
3446         thread_ip_inc(p);
3447 }
3448
3449 static inline void
3450 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3451 {
3452         struct thread *t = &p->threads[p->thread_id];
3453         struct instruction *ip = t->ip;
3454
3455         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3456
3457         /* Structs. */
3458         ALU_MI(t, ip, +);
3459
3460         /* Thread. */
3461         thread_ip_inc(p);
3462 }
3463
3464 static inline void
3465 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3466 {
3467         struct thread *t = &p->threads[p->thread_id];
3468         struct instruction *ip = t->ip;
3469
3470         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3471
3472         /* Structs. */
3473         ALU_HI(t, ip, +);
3474
3475         /* Thread. */
3476         thread_ip_inc(p);
3477 }
3478
3479 static inline void
3480 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3481 {
3482         struct thread *t = &p->threads[p->thread_id];
3483         struct instruction *ip = t->ip;
3484
3485         TRACE("[Thread %2u] sub\n", p->thread_id);
3486
3487         /* Structs. */
3488         ALU(t, ip, -);
3489
3490         /* Thread. */
3491         thread_ip_inc(p);
3492 }
3493
3494 static inline void
3495 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3496 {
3497         struct thread *t = &p->threads[p->thread_id];
3498         struct instruction *ip = t->ip;
3499
3500         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3501
3502         /* Structs. */
3503         ALU_MH(t, ip, -);
3504
3505         /* Thread. */
3506         thread_ip_inc(p);
3507 }
3508
3509 static inline void
3510 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3511 {
3512         struct thread *t = &p->threads[p->thread_id];
3513         struct instruction *ip = t->ip;
3514
3515         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3516
3517         /* Structs. */
3518         ALU_HM(t, ip, -);
3519
3520         /* Thread. */
3521         thread_ip_inc(p);
3522 }
3523
3524 static inline void
3525 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3526 {
3527         struct thread *t = &p->threads[p->thread_id];
3528         struct instruction *ip = t->ip;
3529
3530         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3531
3532         /* Structs. */
3533         ALU_HH(t, ip, -);
3534
3535         /* Thread. */
3536         thread_ip_inc(p);
3537 }
3538
3539 static inline void
3540 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3541 {
3542         struct thread *t = &p->threads[p->thread_id];
3543         struct instruction *ip = t->ip;
3544
3545         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3546
3547         /* Structs. */
3548         ALU_MI(t, ip, -);
3549
3550         /* Thread. */
3551         thread_ip_inc(p);
3552 }
3553
3554 static inline void
3555 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3556 {
3557         struct thread *t = &p->threads[p->thread_id];
3558         struct instruction *ip = t->ip;
3559
3560         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3561
3562         /* Structs. */
3563         ALU_HI(t, ip, -);
3564
3565         /* Thread. */
3566         thread_ip_inc(p);
3567 }
3568
3569 static inline void
3570 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3571 {
3572         struct thread *t = &p->threads[p->thread_id];
3573         struct instruction *ip = t->ip;
3574
3575         TRACE("[Thread %2u] shl\n", p->thread_id);
3576
3577         /* Structs. */
3578         ALU(t, ip, <<);
3579
3580         /* Thread. */
3581         thread_ip_inc(p);
3582 }
3583
3584 static inline void
3585 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3586 {
3587         struct thread *t = &p->threads[p->thread_id];
3588         struct instruction *ip = t->ip;
3589
3590         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3591
3592         /* Structs. */
3593         ALU_MH(t, ip, <<);
3594
3595         /* Thread. */
3596         thread_ip_inc(p);
3597 }
3598
3599 static inline void
3600 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3601 {
3602         struct thread *t = &p->threads[p->thread_id];
3603         struct instruction *ip = t->ip;
3604
3605         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3606
3607         /* Structs. */
3608         ALU_HM(t, ip, <<);
3609
3610         /* Thread. */
3611         thread_ip_inc(p);
3612 }
3613
3614 static inline void
3615 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3616 {
3617         struct thread *t = &p->threads[p->thread_id];
3618         struct instruction *ip = t->ip;
3619
3620         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3621
3622         /* Structs. */
3623         ALU_HH(t, ip, <<);
3624
3625         /* Thread. */
3626         thread_ip_inc(p);
3627 }
3628
3629 static inline void
3630 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3631 {
3632         struct thread *t = &p->threads[p->thread_id];
3633         struct instruction *ip = t->ip;
3634
3635         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3636
3637         /* Structs. */
3638         ALU_MI(t, ip, <<);
3639
3640         /* Thread. */
3641         thread_ip_inc(p);
3642 }
3643
3644 static inline void
3645 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3646 {
3647         struct thread *t = &p->threads[p->thread_id];
3648         struct instruction *ip = t->ip;
3649
3650         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3651
3652         /* Structs. */
3653         ALU_HI(t, ip, <<);
3654
3655         /* Thread. */
3656         thread_ip_inc(p);
3657 }
3658
3659 static inline void
3660 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3661 {
3662         struct thread *t = &p->threads[p->thread_id];
3663         struct instruction *ip = t->ip;
3664
3665         TRACE("[Thread %2u] shr\n", p->thread_id);
3666
3667         /* Structs. */
3668         ALU(t, ip, >>);
3669
3670         /* Thread. */
3671         thread_ip_inc(p);
3672 }
3673
3674 static inline void
3675 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3676 {
3677         struct thread *t = &p->threads[p->thread_id];
3678         struct instruction *ip = t->ip;
3679
3680         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3681
3682         /* Structs. */
3683         ALU_MH(t, ip, >>);
3684
3685         /* Thread. */
3686         thread_ip_inc(p);
3687 }
3688
3689 static inline void
3690 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3691 {
3692         struct thread *t = &p->threads[p->thread_id];
3693         struct instruction *ip = t->ip;
3694
3695         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3696
3697         /* Structs. */
3698         ALU_HM(t, ip, >>);
3699
3700         /* Thread. */
3701         thread_ip_inc(p);
3702 }
3703
3704 static inline void
3705 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3706 {
3707         struct thread *t = &p->threads[p->thread_id];
3708         struct instruction *ip = t->ip;
3709
3710         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3711
3712         /* Structs. */
3713         ALU_HH(t, ip, >>);
3714
3715         /* Thread. */
3716         thread_ip_inc(p);
3717 }
3718
3719 static inline void
3720 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3721 {
3722         struct thread *t = &p->threads[p->thread_id];
3723         struct instruction *ip = t->ip;
3724
3725         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3726
3727         /* Structs. */
3728         ALU_MI(t, ip, >>);
3729
3730         /* Thread. */
3731         thread_ip_inc(p);
3732 }
3733
3734 static inline void
3735 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3736 {
3737         struct thread *t = &p->threads[p->thread_id];
3738         struct instruction *ip = t->ip;
3739
3740         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3741
3742         /* Structs. */
3743         ALU_HI(t, ip, >>);
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 static inline void
3750 instr_alu_and_exec(struct rte_swx_pipeline *p)
3751 {
3752         struct thread *t = &p->threads[p->thread_id];
3753         struct instruction *ip = t->ip;
3754
3755         TRACE("[Thread %2u] and\n", p->thread_id);
3756
3757         /* Structs. */
3758         ALU(t, ip, &);
3759
3760         /* Thread. */
3761         thread_ip_inc(p);
3762 }
3763
3764 static inline void
3765 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3766 {
3767         struct thread *t = &p->threads[p->thread_id];
3768         struct instruction *ip = t->ip;
3769
3770         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3771
3772         /* Structs. */
3773         ALU_MH(t, ip, &);
3774
3775         /* Thread. */
3776         thread_ip_inc(p);
3777 }
3778
3779 static inline void
3780 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3781 {
3782         struct thread *t = &p->threads[p->thread_id];
3783         struct instruction *ip = t->ip;
3784
3785         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3786
3787         /* Structs. */
3788         ALU_HM_FAST(t, ip, &);
3789
3790         /* Thread. */
3791         thread_ip_inc(p);
3792 }
3793
3794 static inline void
3795 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3796 {
3797         struct thread *t = &p->threads[p->thread_id];
3798         struct instruction *ip = t->ip;
3799
3800         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3801
3802         /* Structs. */
3803         ALU_HH_FAST(t, ip, &);
3804
3805         /* Thread. */
3806         thread_ip_inc(p);
3807 }
3808
3809 static inline void
3810 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3811 {
3812         struct thread *t = &p->threads[p->thread_id];
3813         struct instruction *ip = t->ip;
3814
3815         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3816
3817         /* Structs. */
3818         ALU_I(t, ip, &);
3819
3820         /* Thread. */
3821         thread_ip_inc(p);
3822 }
3823
3824 static inline void
3825 instr_alu_or_exec(struct rte_swx_pipeline *p)
3826 {
3827         struct thread *t = &p->threads[p->thread_id];
3828         struct instruction *ip = t->ip;
3829
3830         TRACE("[Thread %2u] or\n", p->thread_id);
3831
3832         /* Structs. */
3833         ALU(t, ip, |);
3834
3835         /* Thread. */
3836         thread_ip_inc(p);
3837 }
3838
3839 static inline void
3840 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3841 {
3842         struct thread *t = &p->threads[p->thread_id];
3843         struct instruction *ip = t->ip;
3844
3845         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3846
3847         /* Structs. */
3848         ALU_MH(t, ip, |);
3849
3850         /* Thread. */
3851         thread_ip_inc(p);
3852 }
3853
3854 static inline void
3855 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3856 {
3857         struct thread *t = &p->threads[p->thread_id];
3858         struct instruction *ip = t->ip;
3859
3860         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3861
3862         /* Structs. */
3863         ALU_HM_FAST(t, ip, |);
3864
3865         /* Thread. */
3866         thread_ip_inc(p);
3867 }
3868
3869 static inline void
3870 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3871 {
3872         struct thread *t = &p->threads[p->thread_id];
3873         struct instruction *ip = t->ip;
3874
3875         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3876
3877         /* Structs. */
3878         ALU_HH_FAST(t, ip, |);
3879
3880         /* Thread. */
3881         thread_ip_inc(p);
3882 }
3883
3884 static inline void
3885 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3886 {
3887         struct thread *t = &p->threads[p->thread_id];
3888         struct instruction *ip = t->ip;
3889
3890         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3891
3892         /* Structs. */
3893         ALU_I(t, ip, |);
3894
3895         /* Thread. */
3896         thread_ip_inc(p);
3897 }
3898
3899 static inline void
3900 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3901 {
3902         struct thread *t = &p->threads[p->thread_id];
3903         struct instruction *ip = t->ip;
3904
3905         TRACE("[Thread %2u] xor\n", p->thread_id);
3906
3907         /* Structs. */
3908         ALU(t, ip, ^);
3909
3910         /* Thread. */
3911         thread_ip_inc(p);
3912 }
3913
3914 static inline void
3915 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3916 {
3917         struct thread *t = &p->threads[p->thread_id];
3918         struct instruction *ip = t->ip;
3919
3920         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3921
3922         /* Structs. */
3923         ALU_MH(t, ip, ^);
3924
3925         /* Thread. */
3926         thread_ip_inc(p);
3927 }
3928
3929 static inline void
3930 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3931 {
3932         struct thread *t = &p->threads[p->thread_id];
3933         struct instruction *ip = t->ip;
3934
3935         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3936
3937         /* Structs. */
3938         ALU_HM_FAST(t, ip, ^);
3939
3940         /* Thread. */
3941         thread_ip_inc(p);
3942 }
3943
3944 static inline void
3945 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3946 {
3947         struct thread *t = &p->threads[p->thread_id];
3948         struct instruction *ip = t->ip;
3949
3950         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3951
3952         /* Structs. */
3953         ALU_HH_FAST(t, ip, ^);
3954
3955         /* Thread. */
3956         thread_ip_inc(p);
3957 }
3958
3959 static inline void
3960 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3961 {
3962         struct thread *t = &p->threads[p->thread_id];
3963         struct instruction *ip = t->ip;
3964
3965         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3966
3967         /* Structs. */
3968         ALU_I(t, ip, ^);
3969
3970         /* Thread. */
3971         thread_ip_inc(p);
3972 }
3973
3974 static inline void
3975 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3976 {
3977         struct thread *t = &p->threads[p->thread_id];
3978         struct instruction *ip = t->ip;
3979         uint8_t *dst_struct, *src_struct;
3980         uint16_t *dst16_ptr, dst;
3981         uint64_t *src64_ptr, src64, src64_mask, src;
3982         uint64_t r;
3983
3984         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3985
3986         /* Structs. */
3987         dst_struct = t->structs[ip->alu.dst.struct_id];
3988         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3989         dst = *dst16_ptr;
3990
3991         src_struct = t->structs[ip->alu.src.struct_id];
3992         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3993         src64 = *src64_ptr;
3994         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3995         src = src64 & src64_mask;
3996
3997         r = dst;
3998         r = ~r & 0xFFFF;
3999
4000         /* The first input (r) is a 16-bit number. The second and the third
4001          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4002          * three numbers (output r) is a 34-bit number.
4003          */
4004         r += (src >> 32) + (src & 0xFFFFFFFF);
4005
4006         /* The first input is a 16-bit number. The second input is an 18-bit
4007          * number. In the worst case scenario, the sum of the two numbers is a
4008          * 19-bit number.
4009          */
4010         r = (r & 0xFFFF) + (r >> 16);
4011
4012         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4013          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4014          */
4015         r = (r & 0xFFFF) + (r >> 16);
4016
4017         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4018          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4019          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4020          * therefore the output r is always a 16-bit number.
4021          */
4022         r = (r & 0xFFFF) + (r >> 16);
4023
4024         r = ~r & 0xFFFF;
4025         r = r ? r : 0xFFFF;
4026
4027         *dst16_ptr = (uint16_t)r;
4028
4029         /* Thread. */
4030         thread_ip_inc(p);
4031 }
4032
4033 static inline void
4034 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4035 {
4036         struct thread *t = &p->threads[p->thread_id];
4037         struct instruction *ip = t->ip;
4038         uint8_t *dst_struct, *src_struct;
4039         uint16_t *dst16_ptr, dst;
4040         uint64_t *src64_ptr, src64, src64_mask, src;
4041         uint64_t r;
4042
4043         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4044
4045         /* Structs. */
4046         dst_struct = t->structs[ip->alu.dst.struct_id];
4047         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4048         dst = *dst16_ptr;
4049
4050         src_struct = t->structs[ip->alu.src.struct_id];
4051         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4052         src64 = *src64_ptr;
4053         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4054         src = src64 & src64_mask;
4055
4056         r = dst;
4057         r = ~r & 0xFFFF;
4058
4059         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4060          * the following sequence of operations in 2's complement arithmetic:
4061          *    a '- b = (a - b) % 0xFFFF.
4062          *
4063          * In order to prevent an underflow for the below subtraction, in which
4064          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4065          * minuend), we first add a multiple of the 0xFFFF modulus to the
4066          * minuend. The number we add to the minuend needs to be a 34-bit number
4067          * or higher, so for readability reasons we picked the 36-bit multiple.
4068          * We are effectively turning the 16-bit minuend into a 36-bit number:
4069          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4070          */
4071         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4072
4073         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4074          * result (the output r) is a 36-bit number.
4075          */
4076         r -= (src >> 32) + (src & 0xFFFFFFFF);
4077
4078         /* The first input is a 16-bit number. The second input is a 20-bit
4079          * number. Their sum is a 21-bit number.
4080          */
4081         r = (r & 0xFFFF) + (r >> 16);
4082
4083         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4084          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4085          */
4086         r = (r & 0xFFFF) + (r >> 16);
4087
4088         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4089          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4090          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4091          * generated, therefore the output r is always a 16-bit number.
4092          */
4093         r = (r & 0xFFFF) + (r >> 16);
4094
4095         r = ~r & 0xFFFF;
4096         r = r ? r : 0xFFFF;
4097
4098         *dst16_ptr = (uint16_t)r;
4099
4100         /* Thread. */
4101         thread_ip_inc(p);
4102 }
4103
4104 static inline void
4105 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4106 {
4107         struct thread *t = &p->threads[p->thread_id];
4108         struct instruction *ip = t->ip;
4109         uint8_t *dst_struct, *src_struct;
4110         uint16_t *dst16_ptr;
4111         uint32_t *src32_ptr;
4112         uint64_t r0, r1;
4113
4114         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4115
4116         /* Structs. */
4117         dst_struct = t->structs[ip->alu.dst.struct_id];
4118         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4119
4120         src_struct = t->structs[ip->alu.src.struct_id];
4121         src32_ptr = (uint32_t *)&src_struct[0];
4122
4123         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4124         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4125         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4126         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4127         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4128
4129         /* The first input is a 16-bit number. The second input is a 19-bit
4130          * number. Their sum is a 20-bit number.
4131          */
4132         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4133
4134         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4135          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4136          */
4137         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4138
4139         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4140          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4141          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4142          * generated, therefore the output r is always a 16-bit number.
4143          */
4144         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4145
4146         r0 = ~r0 & 0xFFFF;
4147         r0 = r0 ? r0 : 0xFFFF;
4148
4149         *dst16_ptr = (uint16_t)r0;
4150
4151         /* Thread. */
4152         thread_ip_inc(p);
4153 }
4154
4155 static inline void
4156 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4157 {
4158         struct thread *t = &p->threads[p->thread_id];
4159         struct instruction *ip = t->ip;
4160         uint8_t *dst_struct, *src_struct;
4161         uint16_t *dst16_ptr;
4162         uint32_t *src32_ptr;
4163         uint64_t r = 0;
4164         uint32_t i;
4165
4166         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4167
4168         /* Structs. */
4169         dst_struct = t->structs[ip->alu.dst.struct_id];
4170         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4171
4172         src_struct = t->structs[ip->alu.src.struct_id];
4173         src32_ptr = (uint32_t *)&src_struct[0];
4174
4175         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4176          * Therefore, in the worst case scenario, a 35-bit number is added to a
4177          * 16-bit number (the input r), so the output r is 36-bit number.
4178          */
4179         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4180                 r += *src32_ptr;
4181
4182         /* The first input is a 16-bit number. The second input is a 20-bit
4183          * number. Their sum is a 21-bit number.
4184          */
4185         r = (r & 0xFFFF) + (r >> 16);
4186
4187         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4188          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4189          */
4190         r = (r & 0xFFFF) + (r >> 16);
4191
4192         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4193          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4194          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4195          * generated, therefore the output r is always a 16-bit number.
4196          */
4197         r = (r & 0xFFFF) + (r >> 16);
4198
4199         r = ~r & 0xFFFF;
4200         r = r ? r : 0xFFFF;
4201
4202         *dst16_ptr = (uint16_t)r;
4203
4204         /* Thread. */
4205         thread_ip_inc(p);
4206 }
4207
4208 /*
4209  * Register array.
4210  */
4211 static struct regarray *
4212 regarray_find(struct rte_swx_pipeline *p, const char *name);
4213
4214 static int
4215 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4216                       struct action *action,
4217                       char **tokens,
4218                       int n_tokens,
4219                       struct instruction *instr,
4220                       struct instruction_data *data __rte_unused)
4221 {
4222         char *regarray = tokens[1], *idx = tokens[2];
4223         struct regarray *r;
4224         struct field *fidx;
4225         uint32_t idx_struct_id, idx_val;
4226
4227         CHECK(n_tokens == 3, EINVAL);
4228
4229         r = regarray_find(p, regarray);
4230         CHECK(r, EINVAL);
4231
4232         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4233         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4234         if (fidx) {
4235                 CHECK(!fidx->var_size, EINVAL);
4236
4237                 instr->type = INSTR_REGPREFETCH_RM;
4238                 if (idx[0] == 'h')
4239                         instr->type = INSTR_REGPREFETCH_RH;
4240
4241                 instr->regarray.regarray_id = r->id;
4242                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4243                 instr->regarray.idx.n_bits = fidx->n_bits;
4244                 instr->regarray.idx.offset = fidx->offset / 8;
4245                 instr->regarray.dstsrc_val = 0; /* Unused. */
4246                 return 0;
4247         }
4248
4249         /* REGPREFETCH_RI. */
4250         idx_val = strtoul(idx, &idx, 0);
4251         CHECK(!idx[0], EINVAL);
4252
4253         instr->type = INSTR_REGPREFETCH_RI;
4254         instr->regarray.regarray_id = r->id;
4255         instr->regarray.idx_val = idx_val;
4256         instr->regarray.dstsrc_val = 0; /* Unused. */
4257         return 0;
4258 }
4259
4260 static int
4261 instr_regrd_translate(struct rte_swx_pipeline *p,
4262                       struct action *action,
4263                       char **tokens,
4264                       int n_tokens,
4265                       struct instruction *instr,
4266                       struct instruction_data *data __rte_unused)
4267 {
4268         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4269         struct regarray *r;
4270         struct field *fdst, *fidx;
4271         uint32_t dst_struct_id, idx_struct_id, idx_val;
4272
4273         CHECK(n_tokens == 4, EINVAL);
4274
4275         r = regarray_find(p, regarray);
4276         CHECK(r, EINVAL);
4277
4278         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4279         CHECK(fdst, EINVAL);
4280         CHECK(!fdst->var_size, EINVAL);
4281
4282         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4283         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4284         if (fidx) {
4285                 CHECK(!fidx->var_size, EINVAL);
4286
4287                 instr->type = INSTR_REGRD_MRM;
4288                 if (dst[0] == 'h' && idx[0] != 'h')
4289                         instr->type = INSTR_REGRD_HRM;
4290                 if (dst[0] != 'h' && idx[0] == 'h')
4291                         instr->type = INSTR_REGRD_MRH;
4292                 if (dst[0] == 'h' && idx[0] == 'h')
4293                         instr->type = INSTR_REGRD_HRH;
4294
4295                 instr->regarray.regarray_id = r->id;
4296                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4297                 instr->regarray.idx.n_bits = fidx->n_bits;
4298                 instr->regarray.idx.offset = fidx->offset / 8;
4299                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4300                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4301                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4302                 return 0;
4303         }
4304
4305         /* REGRD_MRI, REGRD_HRI. */
4306         idx_val = strtoul(idx, &idx, 0);
4307         CHECK(!idx[0], EINVAL);
4308
4309         instr->type = INSTR_REGRD_MRI;
4310         if (dst[0] == 'h')
4311                 instr->type = INSTR_REGRD_HRI;
4312
4313         instr->regarray.regarray_id = r->id;
4314         instr->regarray.idx_val = idx_val;
4315         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4316         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4317         instr->regarray.dstsrc.offset = fdst->offset / 8;
4318         return 0;
4319 }
4320
4321 static int
4322 instr_regwr_translate(struct rte_swx_pipeline *p,
4323                       struct action *action,
4324                       char **tokens,
4325                       int n_tokens,
4326                       struct instruction *instr,
4327                       struct instruction_data *data __rte_unused)
4328 {
4329         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4330         struct regarray *r;
4331         struct field *fidx, *fsrc;
4332         uint64_t src_val;
4333         uint32_t idx_struct_id, idx_val, src_struct_id;
4334
4335         CHECK(n_tokens == 4, EINVAL);
4336
4337         r = regarray_find(p, regarray);
4338         CHECK(r, EINVAL);
4339
4340         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4341         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4342         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4343         if (fidx && fsrc) {
4344                 CHECK(!fidx->var_size, EINVAL);
4345                 CHECK(!fsrc->var_size, EINVAL);
4346
4347                 instr->type = INSTR_REGWR_RMM;
4348                 if (idx[0] == 'h' && src[0] != 'h')
4349                         instr->type = INSTR_REGWR_RHM;
4350                 if (idx[0] != 'h' && src[0] == 'h')
4351                         instr->type = INSTR_REGWR_RMH;
4352                 if (idx[0] == 'h' && src[0] == 'h')
4353                         instr->type = INSTR_REGWR_RHH;
4354
4355                 instr->regarray.regarray_id = r->id;
4356                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4357                 instr->regarray.idx.n_bits = fidx->n_bits;
4358                 instr->regarray.idx.offset = fidx->offset / 8;
4359                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4360                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4361                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4362                 return 0;
4363         }
4364
4365         /* REGWR_RHI, REGWR_RMI. */
4366         if (fidx && !fsrc) {
4367                 CHECK(!fidx->var_size, EINVAL);
4368
4369                 src_val = strtoull(src, &src, 0);
4370                 CHECK(!src[0], EINVAL);
4371
4372                 instr->type = INSTR_REGWR_RMI;
4373                 if (idx[0] == 'h')
4374                         instr->type = INSTR_REGWR_RHI;
4375
4376                 instr->regarray.regarray_id = r->id;
4377                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4378                 instr->regarray.idx.n_bits = fidx->n_bits;
4379                 instr->regarray.idx.offset = fidx->offset / 8;
4380                 instr->regarray.dstsrc_val = src_val;
4381                 return 0;
4382         }
4383
4384         /* REGWR_RIH, REGWR_RIM. */
4385         if (!fidx && fsrc) {
4386                 idx_val = strtoul(idx, &idx, 0);
4387                 CHECK(!idx[0], EINVAL);
4388
4389                 CHECK(!fsrc->var_size, EINVAL);
4390
4391                 instr->type = INSTR_REGWR_RIM;
4392                 if (src[0] == 'h')
4393                         instr->type = INSTR_REGWR_RIH;
4394
4395                 instr->regarray.regarray_id = r->id;
4396                 instr->regarray.idx_val = idx_val;
4397                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4398                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4399                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4400                 return 0;
4401         }
4402
4403         /* REGWR_RII. */
4404         src_val = strtoull(src, &src, 0);
4405         CHECK(!src[0], EINVAL);
4406
4407         idx_val = strtoul(idx, &idx, 0);
4408         CHECK(!idx[0], EINVAL);
4409
4410         instr->type = INSTR_REGWR_RII;
4411         instr->regarray.idx_val = idx_val;
4412         instr->regarray.dstsrc_val = src_val;
4413
4414         return 0;
4415 }
4416
4417 static int
4418 instr_regadd_translate(struct rte_swx_pipeline *p,
4419                        struct action *action,
4420                        char **tokens,
4421                        int n_tokens,
4422                        struct instruction *instr,
4423                        struct instruction_data *data __rte_unused)
4424 {
4425         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4426         struct regarray *r;
4427         struct field *fidx, *fsrc;
4428         uint64_t src_val;
4429         uint32_t idx_struct_id, idx_val, src_struct_id;
4430
4431         CHECK(n_tokens == 4, EINVAL);
4432
4433         r = regarray_find(p, regarray);
4434         CHECK(r, EINVAL);
4435
4436         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4437         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4438         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4439         if (fidx && fsrc) {
4440                 CHECK(!fidx->var_size, EINVAL);
4441                 CHECK(!fsrc->var_size, EINVAL);
4442
4443                 instr->type = INSTR_REGADD_RMM;
4444                 if (idx[0] == 'h' && src[0] != 'h')
4445                         instr->type = INSTR_REGADD_RHM;
4446                 if (idx[0] != 'h' && src[0] == 'h')
4447                         instr->type = INSTR_REGADD_RMH;
4448                 if (idx[0] == 'h' && src[0] == 'h')
4449                         instr->type = INSTR_REGADD_RHH;
4450
4451                 instr->regarray.regarray_id = r->id;
4452                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4453                 instr->regarray.idx.n_bits = fidx->n_bits;
4454                 instr->regarray.idx.offset = fidx->offset / 8;
4455                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4456                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4457                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4458                 return 0;
4459         }
4460
4461         /* REGADD_RHI, REGADD_RMI. */
4462         if (fidx && !fsrc) {
4463                 CHECK(!fidx->var_size, EINVAL);
4464
4465                 src_val = strtoull(src, &src, 0);
4466                 CHECK(!src[0], EINVAL);
4467
4468                 instr->type = INSTR_REGADD_RMI;
4469                 if (idx[0] == 'h')
4470                         instr->type = INSTR_REGADD_RHI;
4471
4472                 instr->regarray.regarray_id = r->id;
4473                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4474                 instr->regarray.idx.n_bits = fidx->n_bits;
4475                 instr->regarray.idx.offset = fidx->offset / 8;
4476                 instr->regarray.dstsrc_val = src_val;
4477                 return 0;
4478         }
4479
4480         /* REGADD_RIH, REGADD_RIM. */
4481         if (!fidx && fsrc) {
4482                 idx_val = strtoul(idx, &idx, 0);
4483                 CHECK(!idx[0], EINVAL);
4484
4485                 CHECK(!fsrc->var_size, EINVAL);
4486
4487                 instr->type = INSTR_REGADD_RIM;
4488                 if (src[0] == 'h')
4489                         instr->type = INSTR_REGADD_RIH;
4490
4491                 instr->regarray.regarray_id = r->id;
4492                 instr->regarray.idx_val = idx_val;
4493                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4494                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4495                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4496                 return 0;
4497         }
4498
4499         /* REGADD_RII. */
4500         src_val = strtoull(src, &src, 0);
4501         CHECK(!src[0], EINVAL);
4502
4503         idx_val = strtoul(idx, &idx, 0);
4504         CHECK(!idx[0], EINVAL);
4505
4506         instr->type = INSTR_REGADD_RII;
4507         instr->regarray.idx_val = idx_val;
4508         instr->regarray.dstsrc_val = src_val;
4509         return 0;
4510 }
4511
4512 static inline uint64_t *
4513 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4514 {
4515         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4516         return r->regarray;
4517 }
4518
4519 static inline uint64_t
4520 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4521 {
4522         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4523
4524         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4525         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4526         uint64_t idx64 = *idx64_ptr;
4527         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4528         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4529
4530         return idx;
4531 }
4532
4533 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4534
4535 static inline uint64_t
4536 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4537 {
4538         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4539
4540         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4541         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4542         uint64_t idx64 = *idx64_ptr;
4543         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4544
4545         return idx;
4546 }
4547
4548 #else
4549
4550 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4551
4552 #endif
4553
4554 static inline uint64_t
4555 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4556 {
4557         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4558
4559         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4560
4561         return idx;
4562 }
4563
4564 static inline uint64_t
4565 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4566 {
4567         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4568         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4569         uint64_t src64 = *src64_ptr;
4570         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4571         uint64_t src = src64 & src64_mask;
4572
4573         return src;
4574 }
4575
4576 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4577
4578 static inline uint64_t
4579 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4580 {
4581         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4582         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4583         uint64_t src64 = *src64_ptr;
4584         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4585
4586         return src;
4587 }
4588
4589 #else
4590
4591 #define instr_regarray_src_nbo instr_regarray_src_hbo
4592
4593 #endif
4594
4595 static inline void
4596 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4597 {
4598         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4599         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4600         uint64_t dst64 = *dst64_ptr;
4601         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4602
4603         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4604
4605 }
4606
4607 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4608
4609 static inline void
4610 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4611 {
4612         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4613         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4614         uint64_t dst64 = *dst64_ptr;
4615         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4616
4617         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4618         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4619 }
4620
4621 #else
4622
4623 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4624
4625 #endif
4626
4627 static inline void
4628 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4629 {
4630         struct thread *t = &p->threads[p->thread_id];
4631         struct instruction *ip = t->ip;
4632         uint64_t *regarray, idx;
4633
4634         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4635
4636         /* Structs. */
4637         regarray = instr_regarray_regarray(p, ip);
4638         idx = instr_regarray_idx_nbo(p, t, ip);
4639         rte_prefetch0(&regarray[idx]);
4640
4641         /* Thread. */
4642         thread_ip_inc(p);
4643 }
4644
4645 static inline void
4646 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4647 {
4648         struct thread *t = &p->threads[p->thread_id];
4649         struct instruction *ip = t->ip;
4650         uint64_t *regarray, idx;
4651
4652         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4653
4654         /* Structs. */
4655         regarray = instr_regarray_regarray(p, ip);
4656         idx = instr_regarray_idx_hbo(p, t, ip);
4657         rte_prefetch0(&regarray[idx]);
4658
4659         /* Thread. */
4660         thread_ip_inc(p);
4661 }
4662
4663 static inline void
4664 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4665 {
4666         struct thread *t = &p->threads[p->thread_id];
4667         struct instruction *ip = t->ip;
4668         uint64_t *regarray, idx;
4669
4670         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4671
4672         /* Structs. */
4673         regarray = instr_regarray_regarray(p, ip);
4674         idx = instr_regarray_idx_imm(p, ip);
4675         rte_prefetch0(&regarray[idx]);
4676
4677         /* Thread. */
4678         thread_ip_inc(p);
4679 }
4680
4681 static inline void
4682 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4683 {
4684         struct thread *t = &p->threads[p->thread_id];
4685         struct instruction *ip = t->ip;
4686         uint64_t *regarray, idx;
4687
4688         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4689
4690         /* Structs. */
4691         regarray = instr_regarray_regarray(p, ip);
4692         idx = instr_regarray_idx_nbo(p, t, ip);
4693         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4694
4695         /* Thread. */
4696         thread_ip_inc(p);
4697 }
4698
4699 static inline void
4700 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4701 {
4702         struct thread *t = &p->threads[p->thread_id];
4703         struct instruction *ip = t->ip;
4704         uint64_t *regarray, idx;
4705
4706         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4707
4708         /* Structs. */
4709         regarray = instr_regarray_regarray(p, ip);
4710         idx = instr_regarray_idx_hbo(p, t, ip);
4711         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4712
4713         /* Thread. */
4714         thread_ip_inc(p);
4715 }
4716
4717 static inline void
4718 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4719 {
4720         struct thread *t = &p->threads[p->thread_id];
4721         struct instruction *ip = t->ip;
4722         uint64_t *regarray, idx;
4723
4724         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4725
4726         /* Structs. */
4727         regarray = instr_regarray_regarray(p, ip);
4728         idx = instr_regarray_idx_nbo(p, t, ip);
4729         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4730
4731         /* Thread. */
4732         thread_ip_inc(p);
4733 }
4734
4735 static inline void
4736 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4737 {
4738         struct thread *t = &p->threads[p->thread_id];
4739         struct instruction *ip = t->ip;
4740         uint64_t *regarray, idx;
4741
4742         /* Structs. */
4743         regarray = instr_regarray_regarray(p, ip);
4744         idx = instr_regarray_idx_hbo(p, t, ip);
4745         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4746
4747         /* Thread. */
4748         thread_ip_inc(p);
4749 }
4750
4751 static inline void
4752 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4753 {
4754         struct thread *t = &p->threads[p->thread_id];
4755         struct instruction *ip = t->ip;
4756         uint64_t *regarray, idx;
4757
4758         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4759
4760         /* Structs. */
4761         regarray = instr_regarray_regarray(p, ip);
4762         idx = instr_regarray_idx_imm(p, ip);
4763         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4764
4765         /* Thread. */
4766         thread_ip_inc(p);
4767 }
4768
4769 static inline void
4770 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4771 {
4772         struct thread *t = &p->threads[p->thread_id];
4773         struct instruction *ip = t->ip;
4774         uint64_t *regarray, idx;
4775
4776         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4777
4778         /* Structs. */
4779         regarray = instr_regarray_regarray(p, ip);
4780         idx = instr_regarray_idx_imm(p, ip);
4781         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4782
4783         /* Thread. */
4784         thread_ip_inc(p);
4785 }
4786
4787 static inline void
4788 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4789 {
4790         struct thread *t = &p->threads[p->thread_id];
4791         struct instruction *ip = t->ip;
4792         uint64_t *regarray, idx, src;
4793
4794         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4795
4796         /* Structs. */
4797         regarray = instr_regarray_regarray(p, ip);
4798         idx = instr_regarray_idx_nbo(p, t, ip);
4799         src = instr_regarray_src_nbo(t, ip);
4800         regarray[idx] = src;
4801
4802         /* Thread. */
4803         thread_ip_inc(p);
4804 }
4805
4806 static inline void
4807 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4808 {
4809         struct thread *t = &p->threads[p->thread_id];
4810         struct instruction *ip = t->ip;
4811         uint64_t *regarray, idx, src;
4812
4813         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4814
4815         /* Structs. */
4816         regarray = instr_regarray_regarray(p, ip);
4817         idx = instr_regarray_idx_nbo(p, t, ip);
4818         src = instr_regarray_src_hbo(t, ip);
4819         regarray[idx] = src;
4820
4821         /* Thread. */
4822         thread_ip_inc(p);
4823 }
4824
4825 static inline void
4826 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4827 {
4828         struct thread *t = &p->threads[p->thread_id];
4829         struct instruction *ip = t->ip;
4830         uint64_t *regarray, idx, src;
4831
4832         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4833
4834         /* Structs. */
4835         regarray = instr_regarray_regarray(p, ip);
4836         idx = instr_regarray_idx_hbo(p, t, ip);
4837         src = instr_regarray_src_nbo(t, ip);
4838         regarray[idx] = src;
4839
4840         /* Thread. */
4841         thread_ip_inc(p);
4842 }
4843
4844 static inline void
4845 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4846 {
4847         struct thread *t = &p->threads[p->thread_id];
4848         struct instruction *ip = t->ip;
4849         uint64_t *regarray, idx, src;
4850
4851         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4852
4853         /* Structs. */
4854         regarray = instr_regarray_regarray(p, ip);
4855         idx = instr_regarray_idx_hbo(p, t, ip);
4856         src = instr_regarray_src_hbo(t, ip);
4857         regarray[idx] = src;
4858
4859         /* Thread. */
4860         thread_ip_inc(p);
4861 }
4862
4863 static inline void
4864 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4865 {
4866         struct thread *t = &p->threads[p->thread_id];
4867         struct instruction *ip = t->ip;
4868         uint64_t *regarray, idx, src;
4869
4870         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4871
4872         /* Structs. */
4873         regarray = instr_regarray_regarray(p, ip);
4874         idx = instr_regarray_idx_nbo(p, t, ip);
4875         src = ip->regarray.dstsrc_val;
4876         regarray[idx] = src;
4877
4878         /* Thread. */
4879         thread_ip_inc(p);
4880 }
4881
4882 static inline void
4883 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4884 {
4885         struct thread *t = &p->threads[p->thread_id];
4886         struct instruction *ip = t->ip;
4887         uint64_t *regarray, idx, src;
4888
4889         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4890
4891         /* Structs. */
4892         regarray = instr_regarray_regarray(p, ip);
4893         idx = instr_regarray_idx_hbo(p, t, ip);
4894         src = ip->regarray.dstsrc_val;
4895         regarray[idx] = src;
4896
4897         /* Thread. */
4898         thread_ip_inc(p);
4899 }
4900
4901 static inline void
4902 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4903 {
4904         struct thread *t = &p->threads[p->thread_id];
4905         struct instruction *ip = t->ip;
4906         uint64_t *regarray, idx, src;
4907
4908         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4909
4910         /* Structs. */
4911         regarray = instr_regarray_regarray(p, ip);
4912         idx = instr_regarray_idx_imm(p, ip);
4913         src = instr_regarray_src_nbo(t, ip);
4914         regarray[idx] = src;
4915
4916         /* Thread. */
4917         thread_ip_inc(p);
4918 }
4919
4920 static inline void
4921 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4922 {
4923         struct thread *t = &p->threads[p->thread_id];
4924         struct instruction *ip = t->ip;
4925         uint64_t *regarray, idx, src;
4926
4927         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4928
4929         /* Structs. */
4930         regarray = instr_regarray_regarray(p, ip);
4931         idx = instr_regarray_idx_imm(p, ip);
4932         src = instr_regarray_src_hbo(t, ip);
4933         regarray[idx] = src;
4934
4935         /* Thread. */
4936         thread_ip_inc(p);
4937 }
4938
4939 static inline void
4940 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4941 {
4942         struct thread *t = &p->threads[p->thread_id];
4943         struct instruction *ip = t->ip;
4944         uint64_t *regarray, idx, src;
4945
4946         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4947
4948         /* Structs. */
4949         regarray = instr_regarray_regarray(p, ip);
4950         idx = instr_regarray_idx_imm(p, ip);
4951         src = ip->regarray.dstsrc_val;
4952         regarray[idx] = src;
4953
4954         /* Thread. */
4955         thread_ip_inc(p);
4956 }
4957
4958 static inline void
4959 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4960 {
4961         struct thread *t = &p->threads[p->thread_id];
4962         struct instruction *ip = t->ip;
4963         uint64_t *regarray, idx, src;
4964
4965         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4966
4967         /* Structs. */
4968         regarray = instr_regarray_regarray(p, ip);
4969         idx = instr_regarray_idx_nbo(p, t, ip);
4970         src = instr_regarray_src_nbo(t, ip);
4971         regarray[idx] += src;
4972
4973         /* Thread. */
4974         thread_ip_inc(p);
4975 }
4976
4977 static inline void
4978 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4979 {
4980         struct thread *t = &p->threads[p->thread_id];
4981         struct instruction *ip = t->ip;
4982         uint64_t *regarray, idx, src;
4983
4984         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4985
4986         /* Structs. */
4987         regarray = instr_regarray_regarray(p, ip);
4988         idx = instr_regarray_idx_nbo(p, t, ip);
4989         src = instr_regarray_src_hbo(t, ip);
4990         regarray[idx] += src;
4991
4992         /* Thread. */
4993         thread_ip_inc(p);
4994 }
4995
4996 static inline void
4997 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4998 {
4999         struct thread *t = &p->threads[p->thread_id];
5000         struct instruction *ip = t->ip;
5001         uint64_t *regarray, idx, src;
5002
5003         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5004
5005         /* Structs. */
5006         regarray = instr_regarray_regarray(p, ip);
5007         idx = instr_regarray_idx_hbo(p, t, ip);
5008         src = instr_regarray_src_nbo(t, ip);
5009         regarray[idx] += src;
5010
5011         /* Thread. */
5012         thread_ip_inc(p);
5013 }
5014
5015 static inline void
5016 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5017 {
5018         struct thread *t = &p->threads[p->thread_id];
5019         struct instruction *ip = t->ip;
5020         uint64_t *regarray, idx, src;
5021
5022         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5023
5024         /* Structs. */
5025         regarray = instr_regarray_regarray(p, ip);
5026         idx = instr_regarray_idx_hbo(p, t, ip);
5027         src = instr_regarray_src_hbo(t, ip);
5028         regarray[idx] += src;
5029
5030         /* Thread. */
5031         thread_ip_inc(p);
5032 }
5033
5034 static inline void
5035 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5036 {
5037         struct thread *t = &p->threads[p->thread_id];
5038         struct instruction *ip = t->ip;
5039         uint64_t *regarray, idx, src;
5040
5041         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5042
5043         /* Structs. */
5044         regarray = instr_regarray_regarray(p, ip);
5045         idx = instr_regarray_idx_nbo(p, t, ip);
5046         src = ip->regarray.dstsrc_val;
5047         regarray[idx] += src;
5048
5049         /* Thread. */
5050         thread_ip_inc(p);
5051 }
5052
5053 static inline void
5054 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5055 {
5056         struct thread *t = &p->threads[p->thread_id];
5057         struct instruction *ip = t->ip;
5058         uint64_t *regarray, idx, src;
5059
5060         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5061
5062         /* Structs. */
5063         regarray = instr_regarray_regarray(p, ip);
5064         idx = instr_regarray_idx_hbo(p, t, ip);
5065         src = ip->regarray.dstsrc_val;
5066         regarray[idx] += src;
5067
5068         /* Thread. */
5069         thread_ip_inc(p);
5070 }
5071
5072 static inline void
5073 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5074 {
5075         struct thread *t = &p->threads[p->thread_id];
5076         struct instruction *ip = t->ip;
5077         uint64_t *regarray, idx, src;
5078
5079         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5080
5081         /* Structs. */
5082         regarray = instr_regarray_regarray(p, ip);
5083         idx = instr_regarray_idx_imm(p, ip);
5084         src = instr_regarray_src_nbo(t, ip);
5085         regarray[idx] += src;
5086
5087         /* Thread. */
5088         thread_ip_inc(p);
5089 }
5090
5091 static inline void
5092 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5093 {
5094         struct thread *t = &p->threads[p->thread_id];
5095         struct instruction *ip = t->ip;
5096         uint64_t *regarray, idx, src;
5097
5098         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5099
5100         /* Structs. */
5101         regarray = instr_regarray_regarray(p, ip);
5102         idx = instr_regarray_idx_imm(p, ip);
5103         src = instr_regarray_src_hbo(t, ip);
5104         regarray[idx] += src;
5105
5106         /* Thread. */
5107         thread_ip_inc(p);
5108 }
5109
5110 static inline void
5111 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5112 {
5113         struct thread *t = &p->threads[p->thread_id];
5114         struct instruction *ip = t->ip;
5115         uint64_t *regarray, idx, src;
5116
5117         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5118
5119         /* Structs. */
5120         regarray = instr_regarray_regarray(p, ip);
5121         idx = instr_regarray_idx_imm(p, ip);
5122         src = ip->regarray.dstsrc_val;
5123         regarray[idx] += src;
5124
5125         /* Thread. */
5126         thread_ip_inc(p);
5127 }
5128
5129 /*
5130  * metarray.
5131  */
5132 static struct metarray *
5133 metarray_find(struct rte_swx_pipeline *p, const char *name);
5134
5135 static int
5136 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5137                             struct action *action,
5138                             char **tokens,
5139                             int n_tokens,
5140                             struct instruction *instr,
5141                             struct instruction_data *data __rte_unused)
5142 {
5143         char *metarray = tokens[1], *idx = tokens[2];
5144         struct metarray *m;
5145         struct field *fidx;
5146         uint32_t idx_struct_id, idx_val;
5147
5148         CHECK(n_tokens == 3, EINVAL);
5149
5150         m = metarray_find(p, metarray);
5151         CHECK(m, EINVAL);
5152
5153         /* METPREFETCH_H, METPREFETCH_M. */
5154         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5155         if (fidx) {
5156                 CHECK(!fidx->var_size, EINVAL);
5157
5158                 instr->type = INSTR_METPREFETCH_M;
5159                 if (idx[0] == 'h')
5160                         instr->type = INSTR_METPREFETCH_H;
5161
5162                 instr->meter.metarray_id = m->id;
5163                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5164                 instr->meter.idx.n_bits = fidx->n_bits;
5165                 instr->meter.idx.offset = fidx->offset / 8;
5166                 return 0;
5167         }
5168
5169         /* METPREFETCH_I. */
5170         idx_val = strtoul(idx, &idx, 0);
5171         CHECK(!idx[0], EINVAL);
5172
5173         instr->type = INSTR_METPREFETCH_I;
5174         instr->meter.metarray_id = m->id;
5175         instr->meter.idx_val = idx_val;
5176         return 0;
5177 }
5178
5179 static int
5180 instr_meter_translate(struct rte_swx_pipeline *p,
5181                       struct action *action,
5182                       char **tokens,
5183                       int n_tokens,
5184                       struct instruction *instr,
5185                       struct instruction_data *data __rte_unused)
5186 {
5187         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5188         char *color_in = tokens[4], *color_out = tokens[5];
5189         struct metarray *m;
5190         struct field *fidx, *flength, *fcin, *fcout;
5191         uint32_t idx_struct_id, length_struct_id;
5192         uint32_t color_in_struct_id, color_out_struct_id;
5193
5194         CHECK(n_tokens == 6, EINVAL);
5195
5196         m = metarray_find(p, metarray);
5197         CHECK(m, EINVAL);
5198
5199         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5200
5201         flength = struct_field_parse(p, action, length, &length_struct_id);
5202         CHECK(flength, EINVAL);
5203         CHECK(!flength->var_size, EINVAL);
5204
5205         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5206
5207         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5208         CHECK(fcout, EINVAL);
5209         CHECK(!fcout->var_size, EINVAL);
5210
5211         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5212         if (fidx && fcin) {
5213                 CHECK(!fidx->var_size, EINVAL);
5214                 CHECK(!fcin->var_size, EINVAL);
5215
5216                 instr->type = INSTR_METER_MMM;
5217                 if (idx[0] == 'h' && length[0] == 'h')
5218                         instr->type = INSTR_METER_HHM;
5219                 if (idx[0] == 'h' && length[0] != 'h')
5220                         instr->type = INSTR_METER_HMM;
5221                 if (idx[0] != 'h' && length[0] == 'h')
5222                         instr->type = INSTR_METER_MHM;
5223
5224                 instr->meter.metarray_id = m->id;
5225
5226                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5227                 instr->meter.idx.n_bits = fidx->n_bits;
5228                 instr->meter.idx.offset = fidx->offset / 8;
5229
5230                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5231                 instr->meter.length.n_bits = flength->n_bits;
5232                 instr->meter.length.offset = flength->offset / 8;
5233
5234                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5235                 instr->meter.color_in.n_bits = fcin->n_bits;
5236                 instr->meter.color_in.offset = fcin->offset / 8;
5237
5238                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5239                 instr->meter.color_out.n_bits = fcout->n_bits;
5240                 instr->meter.color_out.offset = fcout->offset / 8;
5241
5242                 return 0;
5243         }
5244
5245         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5246         if (fidx && !fcin) {
5247                 uint32_t color_in_val;
5248
5249                 CHECK(!fidx->var_size, EINVAL);
5250
5251                 color_in_val = strtoul(color_in, &color_in, 0);
5252                 CHECK(!color_in[0], EINVAL);
5253
5254                 instr->type = INSTR_METER_MMI;
5255                 if (idx[0] == 'h' && length[0] == 'h')
5256                         instr->type = INSTR_METER_HHI;
5257                 if (idx[0] == 'h' && length[0] != 'h')
5258                         instr->type = INSTR_METER_HMI;
5259                 if (idx[0] != 'h' && length[0] == 'h')
5260                         instr->type = INSTR_METER_MHI;
5261
5262                 instr->meter.metarray_id = m->id;
5263
5264                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5265                 instr->meter.idx.n_bits = fidx->n_bits;
5266                 instr->meter.idx.offset = fidx->offset / 8;
5267
5268                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5269                 instr->meter.length.n_bits = flength->n_bits;
5270                 instr->meter.length.offset = flength->offset / 8;
5271
5272                 instr->meter.color_in_val = color_in_val;
5273
5274                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5275                 instr->meter.color_out.n_bits = fcout->n_bits;
5276                 instr->meter.color_out.offset = fcout->offset / 8;
5277
5278                 return 0;
5279         }
5280
5281         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5282         if (!fidx && fcin) {
5283                 uint32_t idx_val;
5284
5285                 idx_val = strtoul(idx, &idx, 0);
5286                 CHECK(!idx[0], EINVAL);
5287
5288                 CHECK(!fcin->var_size, EINVAL);
5289
5290                 instr->type = INSTR_METER_IMM;
5291                 if (length[0] == 'h')
5292                         instr->type = INSTR_METER_IHM;
5293
5294                 instr->meter.metarray_id = m->id;
5295
5296                 instr->meter.idx_val = idx_val;
5297
5298                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5299                 instr->meter.length.n_bits = flength->n_bits;
5300                 instr->meter.length.offset = flength->offset / 8;
5301
5302                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5303                 instr->meter.color_in.n_bits = fcin->n_bits;
5304                 instr->meter.color_in.offset = fcin->offset / 8;
5305
5306                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5307                 instr->meter.color_out.n_bits = fcout->n_bits;
5308                 instr->meter.color_out.offset = fcout->offset / 8;
5309
5310                 return 0;
5311         }
5312
5313         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5314         if (!fidx && !fcin) {
5315                 uint32_t idx_val, color_in_val;
5316
5317                 idx_val = strtoul(idx, &idx, 0);
5318                 CHECK(!idx[0], EINVAL);
5319
5320                 color_in_val = strtoul(color_in, &color_in, 0);
5321                 CHECK(!color_in[0], EINVAL);
5322
5323                 instr->type = INSTR_METER_IMI;
5324                 if (length[0] == 'h')
5325                         instr->type = INSTR_METER_IHI;
5326
5327                 instr->meter.metarray_id = m->id;
5328
5329                 instr->meter.idx_val = idx_val;
5330
5331                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5332                 instr->meter.length.n_bits = flength->n_bits;
5333                 instr->meter.length.offset = flength->offset / 8;
5334
5335                 instr->meter.color_in_val = color_in_val;
5336
5337                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5338                 instr->meter.color_out.n_bits = fcout->n_bits;
5339                 instr->meter.color_out.offset = fcout->offset / 8;
5340
5341                 return 0;
5342         }
5343
5344         CHECK(0, EINVAL);
5345 }
5346
5347 static inline struct meter *
5348 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5349 {
5350         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5351
5352         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5353         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5354         uint64_t idx64 = *idx64_ptr;
5355         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5356         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5357
5358         return &r->metarray[idx];
5359 }
5360
5361 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5362
5363 static inline struct meter *
5364 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5365 {
5366         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5367
5368         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5369         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5370         uint64_t idx64 = *idx64_ptr;
5371         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5372
5373         return &r->metarray[idx];
5374 }
5375
5376 #else
5377
5378 #define instr_meter_idx_nbo instr_meter_idx_hbo
5379
5380 #endif
5381
5382 static inline struct meter *
5383 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5384 {
5385         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5386
5387         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5388
5389         return &r->metarray[idx];
5390 }
5391
5392 static inline uint32_t
5393 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5394 {
5395         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5396         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5397         uint64_t src64 = *src64_ptr;
5398         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5399         uint64_t src = src64 & src64_mask;
5400
5401         return (uint32_t)src;
5402 }
5403
5404 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5405
5406 static inline uint32_t
5407 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5408 {
5409         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5410         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5411         uint64_t src64 = *src64_ptr;
5412         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5413
5414         return (uint32_t)src;
5415 }
5416
5417 #else
5418
5419 #define instr_meter_length_nbo instr_meter_length_hbo
5420
5421 #endif
5422
5423 static inline enum rte_color
5424 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5425 {
5426         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5427         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5428         uint64_t src64 = *src64_ptr;
5429         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5430         uint64_t src = src64 & src64_mask;
5431
5432         return (enum rte_color)src;
5433 }
5434
5435 static inline void
5436 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5437 {
5438         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5439         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5440         uint64_t dst64 = *dst64_ptr;
5441         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5442
5443         uint64_t src = (uint64_t)color_out;
5444
5445         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5446 }
5447
5448 static inline void
5449 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5450 {
5451         struct thread *t = &p->threads[p->thread_id];
5452         struct instruction *ip = t->ip;
5453         struct meter *m;
5454
5455         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5456
5457         /* Structs. */
5458         m = instr_meter_idx_nbo(p, t, ip);
5459         rte_prefetch0(m);
5460
5461         /* Thread. */
5462         thread_ip_inc(p);
5463 }
5464
5465 static inline void
5466 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5467 {
5468         struct thread *t = &p->threads[p->thread_id];
5469         struct instruction *ip = t->ip;
5470         struct meter *m;
5471
5472         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5473
5474         /* Structs. */
5475         m = instr_meter_idx_hbo(p, t, ip);
5476         rte_prefetch0(m);
5477
5478         /* Thread. */
5479         thread_ip_inc(p);
5480 }
5481
5482 static inline void
5483 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5484 {
5485         struct thread *t = &p->threads[p->thread_id];
5486         struct instruction *ip = t->ip;
5487         struct meter *m;
5488
5489         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5490
5491         /* Structs. */
5492         m = instr_meter_idx_imm(p, ip);
5493         rte_prefetch0(m);
5494
5495         /* Thread. */
5496         thread_ip_inc(p);
5497 }
5498
5499 static inline void
5500 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5501 {
5502         struct thread *t = &p->threads[p->thread_id];
5503         struct instruction *ip = t->ip;
5504         struct meter *m;
5505         uint64_t time, n_pkts, n_bytes;
5506         uint32_t length;
5507         enum rte_color color_in, color_out;
5508
5509         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5510
5511         /* Structs. */
5512         m = instr_meter_idx_nbo(p, t, ip);
5513         rte_prefetch0(m->n_pkts);
5514         time = rte_get_tsc_cycles();
5515         length = instr_meter_length_nbo(t, ip);
5516         color_in = instr_meter_color_in_hbo(t, ip);
5517
5518         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5519                 &m->profile->profile,
5520                 time,
5521                 length,
5522                 color_in);
5523
5524         color_out &= m->color_mask;
5525
5526         n_pkts = m->n_pkts[color_out];
5527         n_bytes = m->n_bytes[color_out];
5528
5529         instr_meter_color_out_hbo_set(t, ip, color_out);
5530
5531         m->n_pkts[color_out] = n_pkts + 1;
5532         m->n_bytes[color_out] = n_bytes + length;
5533
5534         /* Thread. */
5535         thread_ip_inc(p);
5536 }
5537
5538 static inline void
5539 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5540 {
5541         struct thread *t = &p->threads[p->thread_id];
5542         struct instruction *ip = t->ip;
5543         struct meter *m;
5544         uint64_t time, n_pkts, n_bytes;
5545         uint32_t length;
5546         enum rte_color color_in, color_out;
5547
5548         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5549
5550         /* Structs. */
5551         m = instr_meter_idx_nbo(p, t, ip);
5552         rte_prefetch0(m->n_pkts);
5553         time = rte_get_tsc_cycles();
5554         length = instr_meter_length_nbo(t, ip);
5555         color_in = (enum rte_color)ip->meter.color_in_val;
5556
5557         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5558                 &m->profile->profile,
5559                 time,
5560                 length,
5561                 color_in);
5562
5563         color_out &= m->color_mask;
5564
5565         n_pkts = m->n_pkts[color_out];
5566         n_bytes = m->n_bytes[color_out];
5567
5568         instr_meter_color_out_hbo_set(t, ip, color_out);
5569
5570         m->n_pkts[color_out] = n_pkts + 1;
5571         m->n_bytes[color_out] = n_bytes + length;
5572
5573         /* Thread. */
5574         thread_ip_inc(p);
5575 }
5576
5577 static inline void
5578 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5579 {
5580         struct thread *t = &p->threads[p->thread_id];
5581         struct instruction *ip = t->ip;
5582         struct meter *m;
5583         uint64_t time, n_pkts, n_bytes;
5584         uint32_t length;
5585         enum rte_color color_in, color_out;
5586
5587         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5588
5589         /* Structs. */
5590         m = instr_meter_idx_nbo(p, t, ip);
5591         rte_prefetch0(m->n_pkts);
5592         time = rte_get_tsc_cycles();
5593         length = instr_meter_length_hbo(t, ip);
5594         color_in = instr_meter_color_in_hbo(t, ip);
5595
5596         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5597                 &m->profile->profile,
5598                 time,
5599                 length,
5600                 color_in);
5601
5602         color_out &= m->color_mask;
5603
5604         n_pkts = m->n_pkts[color_out];
5605         n_bytes = m->n_bytes[color_out];
5606
5607         instr_meter_color_out_hbo_set(t, ip, color_out);
5608
5609         m->n_pkts[color_out] = n_pkts + 1;
5610         m->n_bytes[color_out] = n_bytes + length;
5611
5612         /* Thread. */
5613         thread_ip_inc(p);
5614 }
5615 static inline void
5616 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5617 {
5618         struct thread *t = &p->threads[p->thread_id];
5619         struct instruction *ip = t->ip;
5620         struct meter *m;
5621         uint64_t time, n_pkts, n_bytes;
5622         uint32_t length;
5623         enum rte_color color_in, color_out;
5624
5625         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5626
5627         /* Structs. */
5628         m = instr_meter_idx_nbo(p, t, ip);
5629         rte_prefetch0(m->n_pkts);
5630         time = rte_get_tsc_cycles();
5631         length = instr_meter_length_hbo(t, ip);
5632         color_in = (enum rte_color)ip->meter.color_in_val;
5633
5634         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5635                 &m->profile->profile,
5636                 time,
5637                 length,
5638                 color_in);
5639
5640         color_out &= m->color_mask;
5641
5642         n_pkts = m->n_pkts[color_out];
5643         n_bytes = m->n_bytes[color_out];
5644
5645         instr_meter_color_out_hbo_set(t, ip, color_out);
5646
5647         m->n_pkts[color_out] = n_pkts + 1;
5648         m->n_bytes[color_out] = n_bytes + length;
5649
5650         /* Thread. */
5651         thread_ip_inc(p);
5652 }
5653
5654 static inline void
5655 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5656 {
5657         struct thread *t = &p->threads[p->thread_id];
5658         struct instruction *ip = t->ip;
5659         struct meter *m;
5660         uint64_t time, n_pkts, n_bytes;
5661         uint32_t length;
5662         enum rte_color color_in, color_out;
5663
5664         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5665
5666         /* Structs. */
5667         m = instr_meter_idx_hbo(p, t, ip);
5668         rte_prefetch0(m->n_pkts);
5669         time = rte_get_tsc_cycles();
5670         length = instr_meter_length_nbo(t, ip);
5671         color_in = instr_meter_color_in_hbo(t, ip);
5672
5673         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5674                 &m->profile->profile,
5675                 time,
5676                 length,
5677                 color_in);
5678
5679         color_out &= m->color_mask;
5680
5681         n_pkts = m->n_pkts[color_out];
5682         n_bytes = m->n_bytes[color_out];
5683
5684         instr_meter_color_out_hbo_set(t, ip, color_out);
5685
5686         m->n_pkts[color_out] = n_pkts + 1;
5687         m->n_bytes[color_out] = n_bytes + length;
5688
5689         /* Thread. */
5690         thread_ip_inc(p);
5691 }
5692
5693 static inline void
5694 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5695 {
5696         struct thread *t = &p->threads[p->thread_id];
5697         struct instruction *ip = t->ip;
5698         struct meter *m;
5699         uint64_t time, n_pkts, n_bytes;
5700         uint32_t length;
5701         enum rte_color color_in, color_out;
5702
5703         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5704
5705         /* Structs. */
5706         m = instr_meter_idx_hbo(p, t, ip);
5707         rte_prefetch0(m->n_pkts);
5708         time = rte_get_tsc_cycles();
5709         length = instr_meter_length_nbo(t, ip);
5710         color_in = (enum rte_color)ip->meter.color_in_val;
5711
5712         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5713                 &m->profile->profile,
5714                 time,
5715                 length,
5716                 color_in);
5717
5718         color_out &= m->color_mask;
5719
5720         n_pkts = m->n_pkts[color_out];
5721         n_bytes = m->n_bytes[color_out];
5722
5723         instr_meter_color_out_hbo_set(t, ip, color_out);
5724
5725         m->n_pkts[color_out] = n_pkts + 1;
5726         m->n_bytes[color_out] = n_bytes + length;
5727
5728         /* Thread. */
5729         thread_ip_inc(p);
5730 }
5731
5732 static inline void
5733 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5734 {
5735         struct thread *t = &p->threads[p->thread_id];
5736         struct instruction *ip = t->ip;
5737         struct meter *m;
5738         uint64_t time, n_pkts, n_bytes;
5739         uint32_t length;
5740         enum rte_color color_in, color_out;
5741
5742         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5743
5744         /* Structs. */
5745         m = instr_meter_idx_hbo(p, t, ip);
5746         rte_prefetch0(m->n_pkts);
5747         time = rte_get_tsc_cycles();
5748         length = instr_meter_length_hbo(t, ip);
5749         color_in = instr_meter_color_in_hbo(t, ip);
5750
5751         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5752                 &m->profile->profile,
5753                 time,
5754                 length,
5755                 color_in);
5756
5757         color_out &= m->color_mask;
5758
5759         n_pkts = m->n_pkts[color_out];
5760         n_bytes = m->n_bytes[color_out];
5761
5762         instr_meter_color_out_hbo_set(t, ip, color_out);
5763
5764         m->n_pkts[color_out] = n_pkts + 1;
5765         m->n_bytes[color_out] = n_bytes + length;
5766
5767         /* Thread. */
5768         thread_ip_inc(p);
5769 }
5770
5771 static inline void
5772 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5773 {
5774         struct thread *t = &p->threads[p->thread_id];
5775         struct instruction *ip = t->ip;
5776         struct meter *m;
5777         uint64_t time, n_pkts, n_bytes;
5778         uint32_t length;
5779         enum rte_color color_in, color_out;
5780
5781         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5782
5783         /* Structs. */
5784         m = instr_meter_idx_hbo(p, t, ip);
5785         rte_prefetch0(m->n_pkts);
5786         time = rte_get_tsc_cycles();
5787         length = instr_meter_length_hbo(t, ip);
5788         color_in = (enum rte_color)ip->meter.color_in_val;
5789
5790         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5791                 &m->profile->profile,
5792                 time,
5793                 length,
5794                 color_in);
5795
5796         color_out &= m->color_mask;
5797
5798         n_pkts = m->n_pkts[color_out];
5799         n_bytes = m->n_bytes[color_out];
5800
5801         instr_meter_color_out_hbo_set(t, ip, color_out);
5802
5803         m->n_pkts[color_out] = n_pkts + 1;
5804         m->n_bytes[color_out] = n_bytes + length;
5805
5806         /* Thread. */
5807         thread_ip_inc(p);
5808 }
5809
5810 static inline void
5811 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5812 {
5813         struct thread *t = &p->threads[p->thread_id];
5814         struct instruction *ip = t->ip;
5815         struct meter *m;
5816         uint64_t time, n_pkts, n_bytes;
5817         uint32_t length;
5818         enum rte_color color_in, color_out;
5819
5820         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5821
5822         /* Structs. */
5823         m = instr_meter_idx_imm(p, ip);
5824         rte_prefetch0(m->n_pkts);
5825         time = rte_get_tsc_cycles();
5826         length = instr_meter_length_nbo(t, ip);
5827         color_in = instr_meter_color_in_hbo(t, ip);
5828
5829         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5830                 &m->profile->profile,
5831                 time,
5832                 length,
5833                 color_in);
5834
5835         color_out &= m->color_mask;
5836
5837         n_pkts = m->n_pkts[color_out];
5838         n_bytes = m->n_bytes[color_out];
5839
5840         instr_meter_color_out_hbo_set(t, ip, color_out);
5841
5842         m->n_pkts[color_out] = n_pkts + 1;
5843         m->n_bytes[color_out] = n_bytes + length;
5844
5845         /* Thread. */
5846         thread_ip_inc(p);
5847 }
5848
5849 static inline void
5850 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5851 {
5852         struct thread *t = &p->threads[p->thread_id];
5853         struct instruction *ip = t->ip;
5854         struct meter *m;
5855         uint64_t time, n_pkts, n_bytes;
5856         uint32_t length;
5857         enum rte_color color_in, color_out;
5858
5859         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5860
5861         /* Structs. */
5862         m = instr_meter_idx_imm(p, ip);
5863         rte_prefetch0(m->n_pkts);
5864         time = rte_get_tsc_cycles();
5865         length = instr_meter_length_nbo(t, ip);
5866         color_in = (enum rte_color)ip->meter.color_in_val;
5867
5868         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5869                 &m->profile->profile,
5870                 time,
5871                 length,
5872                 color_in);
5873
5874         color_out &= m->color_mask;
5875
5876         n_pkts = m->n_pkts[color_out];
5877         n_bytes = m->n_bytes[color_out];
5878
5879         instr_meter_color_out_hbo_set(t, ip, color_out);
5880
5881         m->n_pkts[color_out] = n_pkts + 1;
5882         m->n_bytes[color_out] = n_bytes + length;
5883
5884         /* Thread. */
5885         thread_ip_inc(p);
5886 }
5887
5888 static inline void
5889 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5890 {
5891         struct thread *t = &p->threads[p->thread_id];
5892         struct instruction *ip = t->ip;
5893         struct meter *m;
5894         uint64_t time, n_pkts, n_bytes;
5895         uint32_t length;
5896         enum rte_color color_in, color_out;
5897
5898         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5899
5900         /* Structs. */
5901         m = instr_meter_idx_imm(p, ip);
5902         rte_prefetch0(m->n_pkts);
5903         time = rte_get_tsc_cycles();
5904         length = instr_meter_length_hbo(t, ip);
5905         color_in = instr_meter_color_in_hbo(t, ip);
5906
5907         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5908                 &m->profile->profile,
5909                 time,
5910                 length,
5911                 color_in);
5912
5913         color_out &= m->color_mask;
5914
5915         n_pkts = m->n_pkts[color_out];
5916         n_bytes = m->n_bytes[color_out];
5917
5918         instr_meter_color_out_hbo_set(t, ip, color_out);
5919
5920         m->n_pkts[color_out] = n_pkts + 1;
5921         m->n_bytes[color_out] = n_bytes + length;
5922
5923         /* Thread. */
5924         thread_ip_inc(p);
5925 }
5926 static inline void
5927 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5928 {
5929         struct thread *t = &p->threads[p->thread_id];
5930         struct instruction *ip = t->ip;
5931         struct meter *m;
5932         uint64_t time, n_pkts, n_bytes;
5933         uint32_t length;
5934         enum rte_color color_in, color_out;
5935
5936         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5937
5938         /* Structs. */
5939         m = instr_meter_idx_imm(p, ip);
5940         rte_prefetch0(m->n_pkts);
5941         time = rte_get_tsc_cycles();
5942         length = instr_meter_length_hbo(t, ip);
5943         color_in = (enum rte_color)ip->meter.color_in_val;
5944
5945         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5946                 &m->profile->profile,
5947                 time,
5948                 length,
5949                 color_in);
5950
5951         color_out &= m->color_mask;
5952
5953         n_pkts = m->n_pkts[color_out];
5954         n_bytes = m->n_bytes[color_out];
5955
5956         instr_meter_color_out_hbo_set(t, ip, color_out);
5957
5958         m->n_pkts[color_out] = n_pkts + 1;
5959         m->n_bytes[color_out] = n_bytes + length;
5960
5961         /* Thread. */
5962         thread_ip_inc(p);
5963 }
5964
5965 /*
5966  * jmp.
5967  */
5968 static int
5969 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5970                     struct action *action __rte_unused,
5971                     char **tokens,
5972                     int n_tokens,
5973                     struct instruction *instr,
5974                     struct instruction_data *data)
5975 {
5976         CHECK(n_tokens == 2, EINVAL);
5977
5978         strcpy(data->jmp_label, tokens[1]);
5979
5980         instr->type = INSTR_JMP;
5981         instr->jmp.ip = NULL; /* Resolved later. */
5982         return 0;
5983 }
5984
5985 static int
5986 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5987                           struct action *action __rte_unused,
5988                           char **tokens,
5989                           int n_tokens,
5990                           struct instruction *instr,
5991                           struct instruction_data *data)
5992 {
5993         struct header *h;
5994
5995         CHECK(n_tokens == 3, EINVAL);
5996
5997         strcpy(data->jmp_label, tokens[1]);
5998
5999         h = header_parse(p, tokens[2]);
6000         CHECK(h, EINVAL);
6001
6002         instr->type = INSTR_JMP_VALID;
6003         instr->jmp.ip = NULL; /* Resolved later. */
6004         instr->jmp.header_id = h->id;
6005         return 0;
6006 }
6007
6008 static int
6009 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6010                             struct action *action __rte_unused,
6011                             char **tokens,
6012                             int n_tokens,
6013                             struct instruction *instr,
6014                             struct instruction_data *data)
6015 {
6016         struct header *h;
6017
6018         CHECK(n_tokens == 3, EINVAL);
6019
6020         strcpy(data->jmp_label, tokens[1]);
6021
6022         h = header_parse(p, tokens[2]);
6023         CHECK(h, EINVAL);
6024
6025         instr->type = INSTR_JMP_INVALID;
6026         instr->jmp.ip = NULL; /* Resolved later. */
6027         instr->jmp.header_id = h->id;
6028         return 0;
6029 }
6030
6031 static int
6032 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6033                         struct action *action,
6034                         char **tokens,
6035                         int n_tokens,
6036                         struct instruction *instr,
6037                         struct instruction_data *data)
6038 {
6039         CHECK(!action, EINVAL);
6040         CHECK(n_tokens == 2, EINVAL);
6041
6042         strcpy(data->jmp_label, tokens[1]);
6043
6044         instr->type = INSTR_JMP_HIT;
6045         instr->jmp.ip = NULL; /* Resolved later. */
6046         return 0;
6047 }
6048
6049 static int
6050 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6051                          struct action *action,
6052                          char **tokens,
6053                          int n_tokens,
6054                          struct instruction *instr,
6055                          struct instruction_data *data)
6056 {
6057         CHECK(!action, EINVAL);
6058         CHECK(n_tokens == 2, EINVAL);
6059
6060         strcpy(data->jmp_label, tokens[1]);
6061
6062         instr->type = INSTR_JMP_MISS;
6063         instr->jmp.ip = NULL; /* Resolved later. */
6064         return 0;
6065 }
6066
6067 static int
6068 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6069                                struct action *action,
6070                                char **tokens,
6071                                int n_tokens,
6072                                struct instruction *instr,
6073                                struct instruction_data *data)
6074 {
6075         struct action *a;
6076
6077         CHECK(!action, EINVAL);
6078         CHECK(n_tokens == 3, EINVAL);
6079
6080         strcpy(data->jmp_label, tokens[1]);
6081
6082         a = action_find(p, tokens[2]);
6083         CHECK(a, EINVAL);
6084
6085         instr->type = INSTR_JMP_ACTION_HIT;
6086         instr->jmp.ip = NULL; /* Resolved later. */
6087         instr->jmp.action_id = a->id;
6088         return 0;
6089 }
6090
6091 static int
6092 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6093                                 struct action *action,
6094                                 char **tokens,
6095                                 int n_tokens,
6096                                 struct instruction *instr,
6097                                 struct instruction_data *data)
6098 {
6099         struct action *a;
6100
6101         CHECK(!action, EINVAL);
6102         CHECK(n_tokens == 3, EINVAL);
6103
6104         strcpy(data->jmp_label, tokens[1]);
6105
6106         a = action_find(p, tokens[2]);
6107         CHECK(a, EINVAL);
6108
6109         instr->type = INSTR_JMP_ACTION_MISS;
6110         instr->jmp.ip = NULL; /* Resolved later. */
6111         instr->jmp.action_id = a->id;
6112         return 0;
6113 }
6114
6115 static int
6116 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6117                        struct action *action,
6118                        char **tokens,
6119                        int n_tokens,
6120                        struct instruction *instr,
6121                        struct instruction_data *data)
6122 {
6123         char *a = tokens[2], *b = tokens[3];
6124         struct field *fa, *fb;
6125         uint64_t b_val;
6126         uint32_t a_struct_id, b_struct_id;
6127
6128         CHECK(n_tokens == 4, EINVAL);
6129
6130         strcpy(data->jmp_label, tokens[1]);
6131
6132         fa = struct_field_parse(p, action, a, &a_struct_id);
6133         CHECK(fa, EINVAL);
6134         CHECK(!fa->var_size, EINVAL);
6135
6136         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
6137         fb = struct_field_parse(p, action, b, &b_struct_id);
6138         if (fb) {
6139                 CHECK(!fb->var_size, EINVAL);
6140
6141                 instr->type = INSTR_JMP_EQ;
6142                 if (a[0] != 'h' && b[0] == 'h')
6143                         instr->type = INSTR_JMP_EQ_MH;
6144                 if (a[0] == 'h' && b[0] != 'h')
6145                         instr->type = INSTR_JMP_EQ_HM;
6146                 if (a[0] == 'h' && b[0] == 'h')
6147                         instr->type = INSTR_JMP_EQ_HH;
6148                 instr->jmp.ip = NULL; /* Resolved later. */
6149
6150                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6151                 instr->jmp.a.n_bits = fa->n_bits;
6152                 instr->jmp.a.offset = fa->offset / 8;
6153                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6154                 instr->jmp.b.n_bits = fb->n_bits;
6155                 instr->jmp.b.offset = fb->offset / 8;
6156                 return 0;
6157         }
6158
6159         /* JMP_EQ_I. */
6160         b_val = strtoull(b, &b, 0);
6161         CHECK(!b[0], EINVAL);
6162
6163         if (a[0] == 'h')
6164                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6165
6166         instr->type = INSTR_JMP_EQ_I;
6167         instr->jmp.ip = NULL; /* Resolved later. */
6168         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6169         instr->jmp.a.n_bits = fa->n_bits;
6170         instr->jmp.a.offset = fa->offset / 8;
6171         instr->jmp.b_val = b_val;
6172         return 0;
6173 }
6174
6175 static int
6176 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6177                         struct action *action,
6178                         char **tokens,
6179                         int n_tokens,
6180                         struct instruction *instr,
6181                         struct instruction_data *data)
6182 {
6183         char *a = tokens[2], *b = tokens[3];
6184         struct field *fa, *fb;
6185         uint64_t b_val;
6186         uint32_t a_struct_id, b_struct_id;
6187
6188         CHECK(n_tokens == 4, EINVAL);
6189
6190         strcpy(data->jmp_label, tokens[1]);
6191
6192         fa = struct_field_parse(p, action, a, &a_struct_id);
6193         CHECK(fa, EINVAL);
6194         CHECK(!fa->var_size, EINVAL);
6195
6196         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
6197         fb = struct_field_parse(p, action, b, &b_struct_id);
6198         if (fb) {
6199                 CHECK(!fb->var_size, EINVAL);
6200
6201                 instr->type = INSTR_JMP_NEQ;
6202                 if (a[0] != 'h' && b[0] == 'h')
6203                         instr->type = INSTR_JMP_NEQ_MH;
6204                 if (a[0] == 'h' && b[0] != 'h')
6205                         instr->type = INSTR_JMP_NEQ_HM;
6206                 if (a[0] == 'h' && b[0] == 'h')
6207                         instr->type = INSTR_JMP_NEQ_HH;
6208                 instr->jmp.ip = NULL; /* Resolved later. */
6209
6210                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6211                 instr->jmp.a.n_bits = fa->n_bits;
6212                 instr->jmp.a.offset = fa->offset / 8;
6213                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6214                 instr->jmp.b.n_bits = fb->n_bits;
6215                 instr->jmp.b.offset = fb->offset / 8;
6216                 return 0;
6217         }
6218
6219         /* JMP_NEQ_I. */
6220         b_val = strtoull(b, &b, 0);
6221         CHECK(!b[0], EINVAL);
6222
6223         if (a[0] == 'h')
6224                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6225
6226         instr->type = INSTR_JMP_NEQ_I;
6227         instr->jmp.ip = NULL; /* Resolved later. */
6228         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6229         instr->jmp.a.n_bits = fa->n_bits;
6230         instr->jmp.a.offset = fa->offset / 8;
6231         instr->jmp.b_val = b_val;
6232         return 0;
6233 }
6234
6235 static int
6236 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6237                        struct action *action,
6238                        char **tokens,
6239                        int n_tokens,
6240                        struct instruction *instr,
6241                        struct instruction_data *data)
6242 {
6243         char *a = tokens[2], *b = tokens[3];
6244         struct field *fa, *fb;
6245         uint64_t b_val;
6246         uint32_t a_struct_id, b_struct_id;
6247
6248         CHECK(n_tokens == 4, EINVAL);
6249
6250         strcpy(data->jmp_label, tokens[1]);
6251
6252         fa = struct_field_parse(p, action, a, &a_struct_id);
6253         CHECK(fa, EINVAL);
6254         CHECK(!fa->var_size, EINVAL);
6255
6256         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6257         fb = struct_field_parse(p, action, b, &b_struct_id);
6258         if (fb) {
6259                 CHECK(!fb->var_size, EINVAL);
6260
6261                 instr->type = INSTR_JMP_LT;
6262                 if (a[0] == 'h' && b[0] != 'h')
6263                         instr->type = INSTR_JMP_LT_HM;
6264                 if (a[0] != 'h' && b[0] == 'h')
6265                         instr->type = INSTR_JMP_LT_MH;
6266                 if (a[0] == 'h' && b[0] == 'h')
6267                         instr->type = INSTR_JMP_LT_HH;
6268                 instr->jmp.ip = NULL; /* Resolved later. */
6269
6270                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6271                 instr->jmp.a.n_bits = fa->n_bits;
6272                 instr->jmp.a.offset = fa->offset / 8;
6273                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6274                 instr->jmp.b.n_bits = fb->n_bits;
6275                 instr->jmp.b.offset = fb->offset / 8;
6276                 return 0;
6277         }
6278
6279         /* JMP_LT_MI, JMP_LT_HI. */
6280         b_val = strtoull(b, &b, 0);
6281         CHECK(!b[0], EINVAL);
6282
6283         instr->type = INSTR_JMP_LT_MI;
6284         if (a[0] == 'h')
6285                 instr->type = INSTR_JMP_LT_HI;
6286         instr->jmp.ip = NULL; /* Resolved later. */
6287
6288         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6289         instr->jmp.a.n_bits = fa->n_bits;
6290         instr->jmp.a.offset = fa->offset / 8;
6291         instr->jmp.b_val = b_val;
6292         return 0;
6293 }
6294
6295 static int
6296 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6297                        struct action *action,
6298                        char **tokens,
6299                        int n_tokens,
6300                        struct instruction *instr,
6301                        struct instruction_data *data)
6302 {
6303         char *a = tokens[2], *b = tokens[3];
6304         struct field *fa, *fb;
6305         uint64_t b_val;
6306         uint32_t a_struct_id, b_struct_id;
6307
6308         CHECK(n_tokens == 4, EINVAL);
6309
6310         strcpy(data->jmp_label, tokens[1]);
6311
6312         fa = struct_field_parse(p, action, a, &a_struct_id);
6313         CHECK(fa, EINVAL);
6314         CHECK(!fa->var_size, EINVAL);
6315
6316         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6317         fb = struct_field_parse(p, action, b, &b_struct_id);
6318         if (fb) {
6319                 CHECK(!fb->var_size, EINVAL);
6320
6321                 instr->type = INSTR_JMP_GT;
6322                 if (a[0] == 'h' && b[0] != 'h')
6323                         instr->type = INSTR_JMP_GT_HM;
6324                 if (a[0] != 'h' && b[0] == 'h')
6325                         instr->type = INSTR_JMP_GT_MH;
6326                 if (a[0] == 'h' && b[0] == 'h')
6327                         instr->type = INSTR_JMP_GT_HH;
6328                 instr->jmp.ip = NULL; /* Resolved later. */
6329
6330                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6331                 instr->jmp.a.n_bits = fa->n_bits;
6332                 instr->jmp.a.offset = fa->offset / 8;
6333                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6334                 instr->jmp.b.n_bits = fb->n_bits;
6335                 instr->jmp.b.offset = fb->offset / 8;
6336                 return 0;
6337         }
6338
6339         /* JMP_GT_MI, JMP_GT_HI. */
6340         b_val = strtoull(b, &b, 0);
6341         CHECK(!b[0], EINVAL);
6342
6343         instr->type = INSTR_JMP_GT_MI;
6344         if (a[0] == 'h')
6345                 instr->type = INSTR_JMP_GT_HI;
6346         instr->jmp.ip = NULL; /* Resolved later. */
6347
6348         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6349         instr->jmp.a.n_bits = fa->n_bits;
6350         instr->jmp.a.offset = fa->offset / 8;
6351         instr->jmp.b_val = b_val;
6352         return 0;
6353 }
6354
6355 static inline void
6356 instr_jmp_exec(struct rte_swx_pipeline *p)
6357 {
6358         struct thread *t = &p->threads[p->thread_id];
6359         struct instruction *ip = t->ip;
6360
6361         TRACE("[Thread %2u] jmp\n", p->thread_id);
6362
6363         thread_ip_set(t, ip->jmp.ip);
6364 }
6365
6366 static inline void
6367 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6368 {
6369         struct thread *t = &p->threads[p->thread_id];
6370         struct instruction *ip = t->ip;
6371         uint32_t header_id = ip->jmp.header_id;
6372
6373         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6374
6375         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6376 }
6377
6378 static inline void
6379 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6380 {
6381         struct thread *t = &p->threads[p->thread_id];
6382         struct instruction *ip = t->ip;
6383         uint32_t header_id = ip->jmp.header_id;
6384
6385         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6386
6387         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6388 }
6389
6390 static inline void
6391 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6392 {
6393         struct thread *t = &p->threads[p->thread_id];
6394         struct instruction *ip = t->ip;
6395         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6396
6397         TRACE("[Thread %2u] jmph\n", p->thread_id);
6398
6399         t->ip = ip_next[t->hit];
6400 }
6401
6402 static inline void
6403 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6404 {
6405         struct thread *t = &p->threads[p->thread_id];
6406         struct instruction *ip = t->ip;
6407         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6408
6409         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6410
6411         t->ip = ip_next[t->hit];
6412 }
6413
6414 static inline void
6415 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6416 {
6417         struct thread *t = &p->threads[p->thread_id];
6418         struct instruction *ip = t->ip;
6419
6420         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6421
6422         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6423 }
6424
6425 static inline void
6426 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6427 {
6428         struct thread *t = &p->threads[p->thread_id];
6429         struct instruction *ip = t->ip;
6430
6431         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6432
6433         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6434 }
6435
6436 static inline void
6437 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6438 {
6439         struct thread *t = &p->threads[p->thread_id];
6440         struct instruction *ip = t->ip;
6441
6442         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6443
6444         JMP_CMP(t, ip, ==);
6445 }
6446
6447 static inline void
6448 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6449 {
6450         struct thread *t = &p->threads[p->thread_id];
6451         struct instruction *ip = t->ip;
6452
6453         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6454
6455         JMP_CMP_MH(t, ip, ==);
6456 }
6457
6458 static inline void
6459 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6460 {
6461         struct thread *t = &p->threads[p->thread_id];
6462         struct instruction *ip = t->ip;
6463
6464         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6465
6466         JMP_CMP_HM(t, ip, ==);
6467 }
6468
6469 static inline void
6470 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6471 {
6472         struct thread *t = &p->threads[p->thread_id];
6473         struct instruction *ip = t->ip;
6474
6475         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6476
6477         JMP_CMP_HH_FAST(t, ip, ==);
6478 }
6479
6480 static inline void
6481 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6482 {
6483         struct thread *t = &p->threads[p->thread_id];
6484         struct instruction *ip = t->ip;
6485
6486         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6487
6488         JMP_CMP_I(t, ip, ==);
6489 }
6490
6491 static inline void
6492 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6493 {
6494         struct thread *t = &p->threads[p->thread_id];
6495         struct instruction *ip = t->ip;
6496
6497         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6498
6499         JMP_CMP(t, ip, !=);
6500 }
6501
6502 static inline void
6503 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6504 {
6505         struct thread *t = &p->threads[p->thread_id];
6506         struct instruction *ip = t->ip;
6507
6508         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6509
6510         JMP_CMP_MH(t, ip, !=);
6511 }
6512
6513 static inline void
6514 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6515 {
6516         struct thread *t = &p->threads[p->thread_id];
6517         struct instruction *ip = t->ip;
6518
6519         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6520
6521         JMP_CMP_HM(t, ip, !=);
6522 }
6523
6524 static inline void
6525 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6526 {
6527         struct thread *t = &p->threads[p->thread_id];
6528         struct instruction *ip = t->ip;
6529
6530         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6531
6532         JMP_CMP_HH_FAST(t, ip, !=);
6533 }
6534
6535 static inline void
6536 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6537 {
6538         struct thread *t = &p->threads[p->thread_id];
6539         struct instruction *ip = t->ip;
6540
6541         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6542
6543         JMP_CMP_I(t, ip, !=);
6544 }
6545
6546 static inline void
6547 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6548 {
6549         struct thread *t = &p->threads[p->thread_id];
6550         struct instruction *ip = t->ip;
6551
6552         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6553
6554         JMP_CMP(t, ip, <);
6555 }
6556
6557 static inline void
6558 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6559 {
6560         struct thread *t = &p->threads[p->thread_id];
6561         struct instruction *ip = t->ip;
6562
6563         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6564
6565         JMP_CMP_MH(t, ip, <);
6566 }
6567
6568 static inline void
6569 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6570 {
6571         struct thread *t = &p->threads[p->thread_id];
6572         struct instruction *ip = t->ip;
6573
6574         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6575
6576         JMP_CMP_HM(t, ip, <);
6577 }
6578
6579 static inline void
6580 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6581 {
6582         struct thread *t = &p->threads[p->thread_id];
6583         struct instruction *ip = t->ip;
6584
6585         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6586
6587         JMP_CMP_HH(t, ip, <);
6588 }
6589
6590 static inline void
6591 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6592 {
6593         struct thread *t = &p->threads[p->thread_id];
6594         struct instruction *ip = t->ip;
6595
6596         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6597
6598         JMP_CMP_MI(t, ip, <);
6599 }
6600
6601 static inline void
6602 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6603 {
6604         struct thread *t = &p->threads[p->thread_id];
6605         struct instruction *ip = t->ip;
6606
6607         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6608
6609         JMP_CMP_HI(t, ip, <);
6610 }
6611
6612 static inline void
6613 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6614 {
6615         struct thread *t = &p->threads[p->thread_id];
6616         struct instruction *ip = t->ip;
6617
6618         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6619
6620         JMP_CMP(t, ip, >);
6621 }
6622
6623 static inline void
6624 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6625 {
6626         struct thread *t = &p->threads[p->thread_id];
6627         struct instruction *ip = t->ip;
6628
6629         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6630
6631         JMP_CMP_MH(t, ip, >);
6632 }
6633
6634 static inline void
6635 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6636 {
6637         struct thread *t = &p->threads[p->thread_id];
6638         struct instruction *ip = t->ip;
6639
6640         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6641
6642         JMP_CMP_HM(t, ip, >);
6643 }
6644
6645 static inline void
6646 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6647 {
6648         struct thread *t = &p->threads[p->thread_id];
6649         struct instruction *ip = t->ip;
6650
6651         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6652
6653         JMP_CMP_HH(t, ip, >);
6654 }
6655
6656 static inline void
6657 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6658 {
6659         struct thread *t = &p->threads[p->thread_id];
6660         struct instruction *ip = t->ip;
6661
6662         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6663
6664         JMP_CMP_MI(t, ip, >);
6665 }
6666
6667 static inline void
6668 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6669 {
6670         struct thread *t = &p->threads[p->thread_id];
6671         struct instruction *ip = t->ip;
6672
6673         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6674
6675         JMP_CMP_HI(t, ip, >);
6676 }
6677
6678 /*
6679  * return.
6680  */
6681 static int
6682 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6683                        struct action *action,
6684                        char **tokens __rte_unused,
6685                        int n_tokens,
6686                        struct instruction *instr,
6687                        struct instruction_data *data __rte_unused)
6688 {
6689         CHECK(action, EINVAL);
6690         CHECK(n_tokens == 1, EINVAL);
6691
6692         instr->type = INSTR_RETURN;
6693         return 0;
6694 }
6695
6696 static inline void
6697 instr_return_exec(struct rte_swx_pipeline *p)
6698 {
6699         struct thread *t = &p->threads[p->thread_id];
6700
6701         TRACE("[Thread %2u] return\n", p->thread_id);
6702
6703         t->ip = t->ret;
6704 }
6705
6706 static int
6707 instr_translate(struct rte_swx_pipeline *p,
6708                 struct action *action,
6709                 char *string,
6710                 struct instruction *instr,
6711                 struct instruction_data *data)
6712 {
6713         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6714         int n_tokens = 0, tpos = 0;
6715
6716         /* Parse the instruction string into tokens. */
6717         for ( ; ; ) {
6718                 char *token;
6719
6720                 token = strtok_r(string, " \t\v", &string);
6721                 if (!token)
6722                         break;
6723
6724                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6725                 CHECK_NAME(token, EINVAL);
6726
6727                 tokens[n_tokens] = token;
6728                 n_tokens++;
6729         }
6730
6731         CHECK(n_tokens, EINVAL);
6732
6733         /* Handle the optional instruction label. */
6734         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6735                 strcpy(data->label, tokens[0]);
6736
6737                 tpos += 2;
6738                 CHECK(n_tokens - tpos, EINVAL);
6739         }
6740
6741         /* Identify the instruction type. */
6742         if (!strcmp(tokens[tpos], "rx"))
6743                 return instr_rx_translate(p,
6744                                           action,
6745                                           &tokens[tpos],
6746                                           n_tokens - tpos,
6747                                           instr,
6748                                           data);
6749
6750         if (!strcmp(tokens[tpos], "tx"))
6751                 return instr_tx_translate(p,
6752                                           action,
6753                                           &tokens[tpos],
6754                                           n_tokens - tpos,
6755                                           instr,
6756                                           data);
6757
6758         if (!strcmp(tokens[tpos], "drop"))
6759                 return instr_drop_translate(p,
6760                                             action,
6761                                             &tokens[tpos],
6762                                             n_tokens - tpos,
6763                                             instr,
6764                                             data);
6765
6766         if (!strcmp(tokens[tpos], "extract"))
6767                 return instr_hdr_extract_translate(p,
6768                                                    action,
6769                                                    &tokens[tpos],
6770                                                    n_tokens - tpos,
6771                                                    instr,
6772                                                    data);
6773
6774         if (!strcmp(tokens[tpos], "lookahead"))
6775                 return instr_hdr_lookahead_translate(p,
6776                                                      action,
6777                                                      &tokens[tpos],
6778                                                      n_tokens - tpos,
6779                                                      instr,
6780                                                      data);
6781
6782         if (!strcmp(tokens[tpos], "emit"))
6783                 return instr_hdr_emit_translate(p,
6784                                                 action,
6785                                                 &tokens[tpos],
6786                                                 n_tokens - tpos,
6787                                                 instr,
6788                                                 data);
6789
6790         if (!strcmp(tokens[tpos], "validate"))
6791                 return instr_hdr_validate_translate(p,
6792                                                     action,
6793                                                     &tokens[tpos],
6794                                                     n_tokens - tpos,
6795                                                     instr,
6796                                                     data);
6797
6798         if (!strcmp(tokens[tpos], "invalidate"))
6799                 return instr_hdr_invalidate_translate(p,
6800                                                       action,
6801                                                       &tokens[tpos],
6802                                                       n_tokens - tpos,
6803                                                       instr,
6804                                                       data);
6805
6806         if (!strcmp(tokens[tpos], "mov"))
6807                 return instr_mov_translate(p,
6808                                            action,
6809                                            &tokens[tpos],
6810                                            n_tokens - tpos,
6811                                            instr,
6812                                            data);
6813
6814         if (!strcmp(tokens[tpos], "add"))
6815                 return instr_alu_add_translate(p,
6816                                                action,
6817                                                &tokens[tpos],
6818                                                n_tokens - tpos,
6819                                                instr,
6820                                                data);
6821
6822         if (!strcmp(tokens[tpos], "sub"))
6823                 return instr_alu_sub_translate(p,
6824                                                action,
6825                                                &tokens[tpos],
6826                                                n_tokens - tpos,
6827                                                instr,
6828                                                data);
6829
6830         if (!strcmp(tokens[tpos], "ckadd"))
6831                 return instr_alu_ckadd_translate(p,
6832                                                  action,
6833                                                  &tokens[tpos],
6834                                                  n_tokens - tpos,
6835                                                  instr,
6836                                                  data);
6837
6838         if (!strcmp(tokens[tpos], "cksub"))
6839                 return instr_alu_cksub_translate(p,
6840                                                  action,
6841                                                  &tokens[tpos],
6842                                                  n_tokens - tpos,
6843                                                  instr,
6844                                                  data);
6845
6846         if (!strcmp(tokens[tpos], "and"))
6847                 return instr_alu_and_translate(p,
6848                                                action,
6849                                                &tokens[tpos],
6850                                                n_tokens - tpos,
6851                                                instr,
6852                                                data);
6853
6854         if (!strcmp(tokens[tpos], "or"))
6855                 return instr_alu_or_translate(p,
6856                                               action,
6857                                               &tokens[tpos],
6858                                               n_tokens - tpos,
6859                                               instr,
6860                                               data);
6861
6862         if (!strcmp(tokens[tpos], "xor"))
6863                 return instr_alu_xor_translate(p,
6864                                                action,
6865                                                &tokens[tpos],
6866                                                n_tokens - tpos,
6867                                                instr,
6868                                                data);
6869
6870         if (!strcmp(tokens[tpos], "shl"))
6871                 return instr_alu_shl_translate(p,
6872                                                action,
6873                                                &tokens[tpos],
6874                                                n_tokens - tpos,
6875                                                instr,
6876                                                data);
6877
6878         if (!strcmp(tokens[tpos], "shr"))
6879                 return instr_alu_shr_translate(p,
6880                                                action,
6881                                                &tokens[tpos],
6882                                                n_tokens - tpos,
6883                                                instr,
6884                                                data);
6885
6886         if (!strcmp(tokens[tpos], "regprefetch"))
6887                 return instr_regprefetch_translate(p,
6888                                                    action,
6889                                                    &tokens[tpos],
6890                                                    n_tokens - tpos,
6891                                                    instr,
6892                                                    data);
6893
6894         if (!strcmp(tokens[tpos], "regrd"))
6895                 return instr_regrd_translate(p,
6896                                              action,
6897                                              &tokens[tpos],
6898                                              n_tokens - tpos,
6899                                              instr,
6900                                              data);
6901
6902         if (!strcmp(tokens[tpos], "regwr"))
6903                 return instr_regwr_translate(p,
6904                                              action,
6905                                              &tokens[tpos],
6906                                              n_tokens - tpos,
6907                                              instr,
6908                                              data);
6909
6910         if (!strcmp(tokens[tpos], "regadd"))
6911                 return instr_regadd_translate(p,
6912                                               action,
6913                                               &tokens[tpos],
6914                                               n_tokens - tpos,
6915                                               instr,
6916                                               data);
6917
6918         if (!strcmp(tokens[tpos], "metprefetch"))
6919                 return instr_metprefetch_translate(p,
6920                                                    action,
6921                                                    &tokens[tpos],
6922                                                    n_tokens - tpos,
6923                                                    instr,
6924                                                    data);
6925
6926         if (!strcmp(tokens[tpos], "meter"))
6927                 return instr_meter_translate(p,
6928                                              action,
6929                                              &tokens[tpos],
6930                                              n_tokens - tpos,
6931                                              instr,
6932                                              data);
6933
6934         if (!strcmp(tokens[tpos], "table"))
6935                 return instr_table_translate(p,
6936                                              action,
6937                                              &tokens[tpos],
6938                                              n_tokens - tpos,
6939                                              instr,
6940                                              data);
6941
6942         if (!strcmp(tokens[tpos], "learn"))
6943                 return instr_learn_translate(p,
6944                                              action,
6945                                              &tokens[tpos],
6946                                              n_tokens - tpos,
6947                                              instr,
6948                                              data);
6949
6950         if (!strcmp(tokens[tpos], "forget"))
6951                 return instr_forget_translate(p,
6952                                               action,
6953                                               &tokens[tpos],
6954                                               n_tokens - tpos,
6955                                               instr,
6956                                               data);
6957
6958         if (!strcmp(tokens[tpos], "extern"))
6959                 return instr_extern_translate(p,
6960                                               action,
6961                                               &tokens[tpos],
6962                                               n_tokens - tpos,
6963                                               instr,
6964                                               data);
6965
6966         if (!strcmp(tokens[tpos], "jmp"))
6967                 return instr_jmp_translate(p,
6968                                            action,
6969                                            &tokens[tpos],
6970                                            n_tokens - tpos,
6971                                            instr,
6972                                            data);
6973
6974         if (!strcmp(tokens[tpos], "jmpv"))
6975                 return instr_jmp_valid_translate(p,
6976                                                  action,
6977                                                  &tokens[tpos],
6978                                                  n_tokens - tpos,
6979                                                  instr,
6980                                                  data);
6981
6982         if (!strcmp(tokens[tpos], "jmpnv"))
6983                 return instr_jmp_invalid_translate(p,
6984                                                    action,
6985                                                    &tokens[tpos],
6986                                                    n_tokens - tpos,
6987                                                    instr,
6988                                                    data);
6989
6990         if (!strcmp(tokens[tpos], "jmph"))
6991                 return instr_jmp_hit_translate(p,
6992                                                action,
6993                                                &tokens[tpos],
6994                                                n_tokens - tpos,
6995                                                instr,
6996                                                data);
6997
6998         if (!strcmp(tokens[tpos], "jmpnh"))
6999                 return instr_jmp_miss_translate(p,
7000                                                 action,
7001                                                 &tokens[tpos],
7002                                                 n_tokens - tpos,
7003                                                 instr,
7004                                                 data);
7005
7006         if (!strcmp(tokens[tpos], "jmpa"))
7007                 return instr_jmp_action_hit_translate(p,
7008                                                       action,
7009                                                       &tokens[tpos],
7010                                                       n_tokens - tpos,
7011                                                       instr,
7012                                                       data);
7013
7014         if (!strcmp(tokens[tpos], "jmpna"))
7015                 return instr_jmp_action_miss_translate(p,
7016                                                        action,
7017                                                        &tokens[tpos],
7018                                                        n_tokens - tpos,
7019                                                        instr,
7020                                                        data);
7021
7022         if (!strcmp(tokens[tpos], "jmpeq"))
7023                 return instr_jmp_eq_translate(p,
7024                                               action,
7025                                               &tokens[tpos],
7026                                               n_tokens - tpos,
7027                                               instr,
7028                                               data);
7029
7030         if (!strcmp(tokens[tpos], "jmpneq"))
7031                 return instr_jmp_neq_translate(p,
7032                                                action,
7033                                                &tokens[tpos],
7034                                                n_tokens - tpos,
7035                                                instr,
7036                                                data);
7037
7038         if (!strcmp(tokens[tpos], "jmplt"))
7039                 return instr_jmp_lt_translate(p,
7040                                               action,
7041                                               &tokens[tpos],
7042                                               n_tokens - tpos,
7043                                               instr,
7044                                               data);
7045
7046         if (!strcmp(tokens[tpos], "jmpgt"))
7047                 return instr_jmp_gt_translate(p,
7048                                               action,
7049                                               &tokens[tpos],
7050                                               n_tokens - tpos,
7051                                               instr,
7052                                               data);
7053
7054         if (!strcmp(tokens[tpos], "return"))
7055                 return instr_return_translate(p,
7056                                               action,
7057                                               &tokens[tpos],
7058                                               n_tokens - tpos,
7059                                               instr,
7060                                               data);
7061
7062         CHECK(0, EINVAL);
7063 }
7064
7065 static struct instruction_data *
7066 label_find(struct instruction_data *data, uint32_t n, const char *label)
7067 {
7068         uint32_t i;
7069
7070         for (i = 0; i < n; i++)
7071                 if (!strcmp(label, data[i].label))
7072                         return &data[i];
7073
7074         return NULL;
7075 }
7076
7077 static uint32_t
7078 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7079 {
7080         uint32_t count = 0, i;
7081
7082         if (!label[0])
7083                 return 0;
7084
7085         for (i = 0; i < n; i++)
7086                 if (!strcmp(label, data[i].jmp_label))
7087                         count++;
7088
7089         return count;
7090 }
7091
7092 static int
7093 instr_label_check(struct instruction_data *instruction_data,
7094                   uint32_t n_instructions)
7095 {
7096         uint32_t i;
7097
7098         /* Check that all instruction labels are unique. */
7099         for (i = 0; i < n_instructions; i++) {
7100                 struct instruction_data *data = &instruction_data[i];
7101                 char *label = data->label;
7102                 uint32_t j;
7103
7104                 if (!label[0])
7105                         continue;
7106
7107                 for (j = i + 1; j < n_instructions; j++)
7108                         CHECK(strcmp(label, data[j].label), EINVAL);
7109         }
7110
7111         /* Get users for each instruction label. */
7112         for (i = 0; i < n_instructions; i++) {
7113                 struct instruction_data *data = &instruction_data[i];
7114                 char *label = data->label;
7115
7116                 data->n_users = label_is_used(instruction_data,
7117                                               n_instructions,
7118                                               label);
7119         }
7120
7121         return 0;
7122 }
7123
7124 static int
7125 instr_jmp_resolve(struct instruction *instructions,
7126                   struct instruction_data *instruction_data,
7127                   uint32_t n_instructions)
7128 {
7129         uint32_t i;
7130
7131         for (i = 0; i < n_instructions; i++) {
7132                 struct instruction *instr = &instructions[i];
7133                 struct instruction_data *data = &instruction_data[i];
7134                 struct instruction_data *found;
7135
7136                 if (!instruction_is_jmp(instr))
7137                         continue;
7138
7139                 found = label_find(instruction_data,
7140                                    n_instructions,
7141                                    data->jmp_label);
7142                 CHECK(found, EINVAL);
7143
7144                 instr->jmp.ip = &instructions[found - instruction_data];
7145         }
7146
7147         return 0;
7148 }
7149
7150 static int
7151 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7152              struct action *a,
7153              struct instruction *instr,
7154              struct instruction_data *data __rte_unused,
7155              uint32_t n_instructions)
7156 {
7157         if (!a) {
7158                 enum instruction_type type;
7159                 uint32_t i;
7160
7161                 /* Check that the first instruction is rx. */
7162                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7163
7164                 /* Check that there is at least one tx instruction. */
7165                 for (i = 0; i < n_instructions; i++) {
7166                         type = instr[i].type;
7167
7168                         if (instruction_is_tx(type))
7169                                 break;
7170                 }
7171                 CHECK(i < n_instructions, EINVAL);
7172
7173                 /* Check that the last instruction is either tx or unconditional
7174                  * jump.
7175                  */
7176                 type = instr[n_instructions - 1].type;
7177                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
7178         }
7179
7180         if (a) {
7181                 enum instruction_type type;
7182                 uint32_t i;
7183
7184                 /* Check that there is at least one return or tx instruction. */
7185                 for (i = 0; i < n_instructions; i++) {
7186                         type = instr[i].type;
7187
7188                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
7189                                 break;
7190                 }
7191                 CHECK(i < n_instructions, EINVAL);
7192         }
7193
7194         return 0;
7195 }
7196
7197 static uint32_t
7198 instr_compact(struct instruction *instructions,
7199               struct instruction_data *instruction_data,
7200               uint32_t n_instructions)
7201 {
7202         uint32_t i, pos = 0;
7203
7204         /* Eliminate the invalid instructions that have been optimized out. */
7205         for (i = 0; i < n_instructions; i++) {
7206                 struct instruction *instr = &instructions[i];
7207                 struct instruction_data *data = &instruction_data[i];
7208
7209                 if (data->invalid)
7210                         continue;
7211
7212                 if (i != pos) {
7213                         memcpy(&instructions[pos], instr, sizeof(*instr));
7214                         memcpy(&instruction_data[pos], data, sizeof(*data));
7215                 }
7216
7217                 pos++;
7218         }
7219
7220         return pos;
7221 }
7222
7223 static int
7224 instr_pattern_extract_many_search(struct instruction *instr,
7225                                   struct instruction_data *data,
7226                                   uint32_t n_instr,
7227                                   uint32_t *n_pattern_instr)
7228 {
7229         uint32_t i;
7230
7231         for (i = 0; i < n_instr; i++) {
7232                 if (data[i].invalid)
7233                         break;
7234
7235                 if (instr[i].type != INSTR_HDR_EXTRACT)
7236                         break;
7237
7238                 if (i == RTE_DIM(instr->io.hdr.header_id))
7239                         break;
7240
7241                 if (i && data[i].n_users)
7242                         break;
7243         }
7244
7245         if (i < 2)
7246                 return 0;
7247
7248         *n_pattern_instr = i;
7249         return 1;
7250 }
7251
7252 static void
7253 instr_pattern_extract_many_replace(struct instruction *instr,
7254                                    struct instruction_data *data,
7255                                    uint32_t n_instr)
7256 {
7257         uint32_t i;
7258
7259         for (i = 1; i < n_instr; i++) {
7260                 instr[0].type++;
7261                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7262                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7263                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7264
7265                 data[i].invalid = 1;
7266         }
7267 }
7268
7269 static uint32_t
7270 instr_pattern_extract_many_optimize(struct instruction *instructions,
7271                                     struct instruction_data *instruction_data,
7272                                     uint32_t n_instructions)
7273 {
7274         uint32_t i;
7275
7276         for (i = 0; i < n_instructions; ) {
7277                 struct instruction *instr = &instructions[i];
7278                 struct instruction_data *data = &instruction_data[i];
7279                 uint32_t n_instr = 0;
7280                 int detected;
7281
7282                 /* Extract many. */
7283                 detected = instr_pattern_extract_many_search(instr,
7284                                                              data,
7285                                                              n_instructions - i,
7286                                                              &n_instr);
7287                 if (detected) {
7288                         instr_pattern_extract_many_replace(instr,
7289                                                            data,
7290                                                            n_instr);
7291                         i += n_instr;
7292                         continue;
7293                 }
7294
7295                 /* No pattern starting at the current instruction. */
7296                 i++;
7297         }
7298
7299         /* Eliminate the invalid instructions that have been optimized out. */
7300         n_instructions = instr_compact(instructions,
7301                                        instruction_data,
7302                                        n_instructions);
7303
7304         return n_instructions;
7305 }
7306
7307 static int
7308 instr_pattern_emit_many_tx_search(struct instruction *instr,
7309                                   struct instruction_data *data,
7310                                   uint32_t n_instr,
7311                                   uint32_t *n_pattern_instr)
7312 {
7313         uint32_t i;
7314
7315         for (i = 0; i < n_instr; i++) {
7316                 if (data[i].invalid)
7317                         break;
7318
7319                 if (instr[i].type != INSTR_HDR_EMIT)
7320                         break;
7321
7322                 if (i == RTE_DIM(instr->io.hdr.header_id))
7323                         break;
7324
7325                 if (i && data[i].n_users)
7326                         break;
7327         }
7328
7329         if (!i)
7330                 return 0;
7331
7332         if (!instruction_is_tx(instr[i].type))
7333                 return 0;
7334
7335         if (data[i].n_users)
7336                 return 0;
7337
7338         i++;
7339
7340         *n_pattern_instr = i;
7341         return 1;
7342 }
7343
7344 static void
7345 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7346                                    struct instruction_data *data,
7347                                    uint32_t n_instr)
7348 {
7349         uint32_t i;
7350
7351         /* Any emit instruction in addition to the first one. */
7352         for (i = 1; i < n_instr - 1; i++) {
7353                 instr[0].type++;
7354                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7355                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7356                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7357
7358                 data[i].invalid = 1;
7359         }
7360
7361         /* The TX instruction is the last one in the pattern. */
7362         instr[0].type++;
7363         instr[0].io.io.offset = instr[i].io.io.offset;
7364         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7365         data[i].invalid = 1;
7366 }
7367
7368 static uint32_t
7369 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7370                                     struct instruction_data *instruction_data,
7371                                     uint32_t n_instructions)
7372 {
7373         uint32_t i;
7374
7375         for (i = 0; i < n_instructions; ) {
7376                 struct instruction *instr = &instructions[i];
7377                 struct instruction_data *data = &instruction_data[i];
7378                 uint32_t n_instr = 0;
7379                 int detected;
7380
7381                 /* Emit many + TX. */
7382                 detected = instr_pattern_emit_many_tx_search(instr,
7383                                                              data,
7384                                                              n_instructions - i,
7385                                                              &n_instr);
7386                 if (detected) {
7387                         instr_pattern_emit_many_tx_replace(instr,
7388                                                            data,
7389                                                            n_instr);
7390                         i += n_instr;
7391                         continue;
7392                 }
7393
7394                 /* No pattern starting at the current instruction. */
7395                 i++;
7396         }
7397
7398         /* Eliminate the invalid instructions that have been optimized out. */
7399         n_instructions = instr_compact(instructions,
7400                                        instruction_data,
7401                                        n_instructions);
7402
7403         return n_instructions;
7404 }
7405
7406 static uint32_t
7407 action_arg_src_mov_count(struct action *a,
7408                          uint32_t arg_id,
7409                          struct instruction *instructions,
7410                          struct instruction_data *instruction_data,
7411                          uint32_t n_instructions);
7412
7413 static int
7414 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7415                                       struct action *a,
7416                                       struct instruction *instr,
7417                                       struct instruction_data *data,
7418                                       uint32_t n_instr,
7419                                       struct instruction *instructions,
7420                                       struct instruction_data *instruction_data,
7421                                       uint32_t n_instructions,
7422                                       uint32_t *n_pattern_instr)
7423 {
7424         struct header *h;
7425         uint32_t src_field_id, i, j;
7426
7427         /* Prerequisites. */
7428         if (!a || !a->st)
7429                 return 0;
7430
7431         /* First instruction: MOV_HM. */
7432         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7433                 return 0;
7434
7435         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7436         if (!h || h->st->var_size)
7437                 return 0;
7438
7439         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7440                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7441                         break;
7442
7443         if (src_field_id == a->st->n_fields)
7444                 return 0;
7445
7446         if (instr[0].mov.dst.offset ||
7447             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7448             instr[0].mov.src.struct_id ||
7449             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7450             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7451                 return 0;
7452
7453         if ((n_instr < h->st->n_fields + 1) ||
7454              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7455                 return 0;
7456
7457         /* Subsequent instructions: MOV_HM. */
7458         for (i = 1; i < h->st->n_fields; i++)
7459                 if (data[i].invalid ||
7460                     data[i].n_users ||
7461                     (instr[i].type != INSTR_MOV_HM) ||
7462                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7463                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7464                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7465                     instr[i].mov.src.struct_id ||
7466                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7467                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7468                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7469                         return 0;
7470
7471         /* Last instruction: HDR_VALIDATE. */
7472         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7473             (instr[i].valid.header_id != h->id))
7474                 return 0;
7475
7476         /* Check that none of the action args that are used as source for this
7477          * DMA transfer are not used as source in any other mov instruction.
7478          */
7479         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7480                 uint32_t n_users;
7481
7482                 n_users = action_arg_src_mov_count(a,
7483                                                    j,
7484                                                    instructions,
7485                                                    instruction_data,
7486                                                    n_instructions);
7487                 if (n_users > 1)
7488                         return 0;
7489         }
7490
7491         *n_pattern_instr = 1 + i;
7492         return 1;
7493 }
7494
7495 static void
7496 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7497                                        struct action *a,
7498                                        struct instruction *instr,
7499                                        struct instruction_data *data,
7500                                        uint32_t n_instr)
7501 {
7502         struct header *h;
7503         uint32_t src_field_id, src_offset, i;
7504
7505         /* Read from the instructions before they are modified. */
7506         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7507         if (!h)
7508                 return;
7509
7510         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7511                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7512                         break;
7513
7514         if (src_field_id == a->st->n_fields)
7515                 return;
7516
7517         src_offset = instr[0].mov.src.offset;
7518
7519         /* Modify the instructions. */
7520         instr[0].type = INSTR_DMA_HT;
7521         instr[0].dma.dst.header_id[0] = h->id;
7522         instr[0].dma.dst.struct_id[0] = h->struct_id;
7523         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7524         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7525
7526         for (i = 1; i < n_instr; i++)
7527                 data[i].invalid = 1;
7528
7529         /* Update the endianness of the action arguments to header endianness. */
7530         for (i = 0; i < h->st->n_fields; i++)
7531                 a->args_endianness[src_field_id + i] = 1;
7532 }
7533
7534 static uint32_t
7535 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7536                                         struct action *a,
7537                                         struct instruction *instructions,
7538                                         struct instruction_data *instruction_data,
7539                                         uint32_t n_instructions)
7540 {
7541         uint32_t i;
7542
7543         if (!a || !a->st)
7544                 return n_instructions;
7545
7546         for (i = 0; i < n_instructions; ) {
7547                 struct instruction *instr = &instructions[i];
7548                 struct instruction_data *data = &instruction_data[i];
7549                 uint32_t n_instr = 0;
7550                 int detected;
7551
7552                 /* Mov all + validate. */
7553                 detected = instr_pattern_mov_all_validate_search(p,
7554                                                                  a,
7555                                                                  instr,
7556                                                                  data,
7557                                                                  n_instructions - i,
7558                                                                  instructions,
7559                                                                  instruction_data,
7560                                                                  n_instructions,
7561                                                                  &n_instr);
7562                 if (detected) {
7563                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7564                         i += n_instr;
7565                         continue;
7566                 }
7567
7568                 /* No pattern starting at the current instruction. */
7569                 i++;
7570         }
7571
7572         /* Eliminate the invalid instructions that have been optimized out. */
7573         n_instructions = instr_compact(instructions,
7574                                        instruction_data,
7575                                        n_instructions);
7576
7577         return n_instructions;
7578 }
7579
7580 static int
7581 instr_pattern_dma_many_search(struct instruction *instr,
7582                               struct instruction_data *data,
7583                               uint32_t n_instr,
7584                               uint32_t *n_pattern_instr)
7585 {
7586         uint32_t i;
7587
7588         for (i = 0; i < n_instr; i++) {
7589                 if (data[i].invalid)
7590                         break;
7591
7592                 if (instr[i].type != INSTR_DMA_HT)
7593                         break;
7594
7595                 if (i == RTE_DIM(instr->dma.dst.header_id))
7596                         break;
7597
7598                 if (i && data[i].n_users)
7599                         break;
7600         }
7601
7602         if (i < 2)
7603                 return 0;
7604
7605         *n_pattern_instr = i;
7606         return 1;
7607 }
7608
7609 static void
7610 instr_pattern_dma_many_replace(struct instruction *instr,
7611                                struct instruction_data *data,
7612                                uint32_t n_instr)
7613 {
7614         uint32_t i;
7615
7616         for (i = 1; i < n_instr; i++) {
7617                 instr[0].type++;
7618                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7619                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7620                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7621                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7622
7623                 data[i].invalid = 1;
7624         }
7625 }
7626
7627 static uint32_t
7628 instr_pattern_dma_many_optimize(struct instruction *instructions,
7629                struct instruction_data *instruction_data,
7630                uint32_t n_instructions)
7631 {
7632         uint32_t i;
7633
7634         for (i = 0; i < n_instructions; ) {
7635                 struct instruction *instr = &instructions[i];
7636                 struct instruction_data *data = &instruction_data[i];
7637                 uint32_t n_instr = 0;
7638                 int detected;
7639
7640                 /* DMA many. */
7641                 detected = instr_pattern_dma_many_search(instr,
7642                                                          data,
7643                                                          n_instructions - i,
7644                                                          &n_instr);
7645                 if (detected) {
7646                         instr_pattern_dma_many_replace(instr, data, n_instr);
7647                         i += n_instr;
7648                         continue;
7649                 }
7650
7651                 /* No pattern starting at the current instruction. */
7652                 i++;
7653         }
7654
7655         /* Eliminate the invalid instructions that have been optimized out. */
7656         n_instructions = instr_compact(instructions,
7657                                        instruction_data,
7658                                        n_instructions);
7659
7660         return n_instructions;
7661 }
7662
7663 static uint32_t
7664 instr_optimize(struct rte_swx_pipeline *p,
7665                struct action *a,
7666                struct instruction *instructions,
7667                struct instruction_data *instruction_data,
7668                uint32_t n_instructions)
7669 {
7670         /* Extract many. */
7671         n_instructions = instr_pattern_extract_many_optimize(instructions,
7672                                                              instruction_data,
7673                                                              n_instructions);
7674
7675         /* Emit many + TX. */
7676         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7677                                                              instruction_data,
7678                                                              n_instructions);
7679
7680         /* Mov all + validate. */
7681         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7682                                                                  a,
7683                                                                  instructions,
7684                                                                  instruction_data,
7685                                                                  n_instructions);
7686
7687         /* DMA many. */
7688         n_instructions = instr_pattern_dma_many_optimize(instructions,
7689                                                          instruction_data,
7690                                                          n_instructions);
7691
7692         return n_instructions;
7693 }
7694
7695 static int
7696 instruction_config(struct rte_swx_pipeline *p,
7697                    struct action *a,
7698                    const char **instructions,
7699                    uint32_t n_instructions)
7700 {
7701         struct instruction *instr = NULL;
7702         struct instruction_data *data = NULL;
7703         int err = 0;
7704         uint32_t i;
7705
7706         CHECK(n_instructions, EINVAL);
7707         CHECK(instructions, EINVAL);
7708         for (i = 0; i < n_instructions; i++)
7709                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7710
7711         /* Memory allocation. */
7712         instr = calloc(n_instructions, sizeof(struct instruction));
7713         if (!instr) {
7714                 err = -ENOMEM;
7715                 goto error;
7716         }
7717
7718         data = calloc(n_instructions, sizeof(struct instruction_data));
7719         if (!data) {
7720                 err = -ENOMEM;
7721                 goto error;
7722         }
7723
7724         for (i = 0; i < n_instructions; i++) {
7725                 char *string = strdup(instructions[i]);
7726                 if (!string) {
7727                         err = -ENOMEM;
7728                         goto error;
7729                 }
7730
7731                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7732                 if (err) {
7733                         free(string);
7734                         goto error;
7735                 }
7736
7737                 free(string);
7738         }
7739
7740         err = instr_label_check(data, n_instructions);
7741         if (err)
7742                 goto error;
7743
7744         err = instr_verify(p, a, instr, data, n_instructions);
7745         if (err)
7746                 goto error;
7747
7748         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7749
7750         err = instr_jmp_resolve(instr, data, n_instructions);
7751         if (err)
7752                 goto error;
7753
7754         if (a) {
7755                 a->instructions = instr;
7756                 a->n_instructions = n_instructions;
7757         } else {
7758                 p->instructions = instr;
7759                 p->n_instructions = n_instructions;
7760         }
7761
7762         free(data);
7763         return 0;
7764
7765 error:
7766         free(data);
7767         free(instr);
7768         return err;
7769 }
7770
7771 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7772
7773 static instr_exec_t instruction_table[] = {
7774         [INSTR_RX] = instr_rx_exec,
7775         [INSTR_TX] = instr_tx_exec,
7776         [INSTR_TX_I] = instr_tx_i_exec,
7777
7778         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7779         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7780         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7781         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7782         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7783         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7784         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7785         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7786         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7787         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7788
7789         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7790         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7791         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7792         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7793         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7794         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7795         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7796         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7797         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7798
7799         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7800         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7801
7802         [INSTR_MOV] = instr_mov_exec,
7803         [INSTR_MOV_MH] = instr_mov_mh_exec,
7804         [INSTR_MOV_HM] = instr_mov_hm_exec,
7805         [INSTR_MOV_HH] = instr_mov_hh_exec,
7806         [INSTR_MOV_I] = instr_mov_i_exec,
7807
7808         [INSTR_DMA_HT] = instr_dma_ht_exec,
7809         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7810         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7811         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7812         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7813         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7814         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7815         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7816
7817         [INSTR_ALU_ADD] = instr_alu_add_exec,
7818         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7819         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7820         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7821         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7822         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7823
7824         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7825         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7826         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7827         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7828         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7829         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7830
7831         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7832         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7833         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7834         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7835
7836         [INSTR_ALU_AND] = instr_alu_and_exec,
7837         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7838         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7839         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7840         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7841
7842         [INSTR_ALU_OR] = instr_alu_or_exec,
7843         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7844         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7845         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7846         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7847
7848         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7849         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7850         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7851         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7852         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7853
7854         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7855         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7856         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7857         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7858         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7859         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7860
7861         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7862         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7863         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7864         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7865         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7866         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7867
7868         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7869         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7870         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7871
7872         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7873         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7874         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7875         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7876         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7877         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7878
7879         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7880         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7881         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7882         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7883         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7884         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7885         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7886         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7887         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7888
7889         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7890         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7891         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7892         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7893         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7894         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7895         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7896         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7897         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7898
7899         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7900         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7901         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7902
7903         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7904         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7905         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7906         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7907         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7908         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7909         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7910         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7911         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7912         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7913         [INSTR_METER_IMM] = instr_meter_imm_exec,
7914         [INSTR_METER_IMI] = instr_meter_imi_exec,
7915
7916         [INSTR_TABLE] = instr_table_exec,
7917         [INSTR_SELECTOR] = instr_selector_exec,
7918         [INSTR_LEARNER] = instr_learner_exec,
7919         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7920         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7921         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7922         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7923
7924         [INSTR_JMP] = instr_jmp_exec,
7925         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7926         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7927         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7928         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7929         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7930         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7931
7932         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7933         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7934         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7935         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7936         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7937
7938         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7939         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7940         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7941         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7942         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7943
7944         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7945         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7946         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7947         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7948         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7949         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7950
7951         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7952         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7953         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7954         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7955         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7956         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7957
7958         [INSTR_RETURN] = instr_return_exec,
7959 };
7960
7961 static inline void
7962 instr_exec(struct rte_swx_pipeline *p)
7963 {
7964         struct thread *t = &p->threads[p->thread_id];
7965         struct instruction *ip = t->ip;
7966         instr_exec_t instr = instruction_table[ip->type];
7967
7968         instr(p);
7969 }
7970
7971 /*
7972  * Action.
7973  */
7974 static struct action *
7975 action_find(struct rte_swx_pipeline *p, const char *name)
7976 {
7977         struct action *elem;
7978
7979         if (!name)
7980                 return NULL;
7981
7982         TAILQ_FOREACH(elem, &p->actions, node)
7983                 if (strcmp(elem->name, name) == 0)
7984                         return elem;
7985
7986         return NULL;
7987 }
7988
7989 static struct action *
7990 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7991 {
7992         struct action *action = NULL;
7993
7994         TAILQ_FOREACH(action, &p->actions, node)
7995                 if (action->id == id)
7996                         return action;
7997
7998         return NULL;
7999 }
8000
8001 static struct field *
8002 action_field_find(struct action *a, const char *name)
8003 {
8004         return a->st ? struct_type_field_find(a->st, name) : NULL;
8005 }
8006
8007 static struct field *
8008 action_field_parse(struct action *action, const char *name)
8009 {
8010         if (name[0] != 't' || name[1] != '.')
8011                 return NULL;
8012
8013         return action_field_find(action, &name[2]);
8014 }
8015
8016 static int
8017 action_has_nbo_args(struct action *a)
8018 {
8019         uint32_t i;
8020
8021         /* Return if the action does not have any args. */
8022         if (!a->st)
8023                 return 0; /* FALSE */
8024
8025         for (i = 0; i < a->st->n_fields; i++)
8026                 if (a->args_endianness[i])
8027                         return 1; /* TRUE */
8028
8029         return 0; /* FALSE */
8030 }
8031
8032 static int
8033 action_does_learning(struct action *a)
8034 {
8035         uint32_t i;
8036
8037         for (i = 0; i < a->n_instructions; i++)
8038                 switch (a->instructions[i].type) {
8039                 case INSTR_LEARNER_LEARN:
8040                         return 1; /* TRUE */
8041
8042                 case INSTR_LEARNER_FORGET:
8043                         return 1; /* TRUE */
8044
8045                 default:
8046                         continue;
8047                 }
8048
8049         return 0; /* FALSE */
8050 }
8051
8052 int
8053 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8054                                const char *name,
8055                                const char *args_struct_type_name,
8056                                const char **instructions,
8057                                uint32_t n_instructions)
8058 {
8059         struct struct_type *args_struct_type = NULL;
8060         struct action *a;
8061         int err;
8062
8063         CHECK(p, EINVAL);
8064
8065         CHECK_NAME(name, EINVAL);
8066         CHECK(!action_find(p, name), EEXIST);
8067
8068         if (args_struct_type_name) {
8069                 CHECK_NAME(args_struct_type_name, EINVAL);
8070                 args_struct_type = struct_type_find(p, args_struct_type_name);
8071                 CHECK(args_struct_type, EINVAL);
8072                 CHECK(!args_struct_type->var_size, EINVAL);
8073         }
8074
8075         /* Node allocation. */
8076         a = calloc(1, sizeof(struct action));
8077         CHECK(a, ENOMEM);
8078         if (args_struct_type) {
8079                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
8080                 if (!a->args_endianness) {
8081                         free(a);
8082                         CHECK(0, ENOMEM);
8083                 }
8084         }
8085
8086         /* Node initialization. */
8087         strcpy(a->name, name);
8088         a->st = args_struct_type;
8089         a->id = p->n_actions;
8090
8091         /* Instruction translation. */
8092         err = instruction_config(p, a, instructions, n_instructions);
8093         if (err) {
8094                 free(a->args_endianness);
8095                 free(a);
8096                 return err;
8097         }
8098
8099         /* Node add to tailq. */
8100         TAILQ_INSERT_TAIL(&p->actions, a, node);
8101         p->n_actions++;
8102
8103         return 0;
8104 }
8105
8106 static int
8107 action_build(struct rte_swx_pipeline *p)
8108 {
8109         struct action *action;
8110
8111         p->action_instructions = calloc(p->n_actions,
8112                                         sizeof(struct instruction *));
8113         CHECK(p->action_instructions, ENOMEM);
8114
8115         TAILQ_FOREACH(action, &p->actions, node)
8116                 p->action_instructions[action->id] = action->instructions;
8117
8118         return 0;
8119 }
8120
8121 static void
8122 action_build_free(struct rte_swx_pipeline *p)
8123 {
8124         free(p->action_instructions);
8125         p->action_instructions = NULL;
8126 }
8127
8128 static void
8129 action_free(struct rte_swx_pipeline *p)
8130 {
8131         action_build_free(p);
8132
8133         for ( ; ; ) {
8134                 struct action *action;
8135
8136                 action = TAILQ_FIRST(&p->actions);
8137                 if (!action)
8138                         break;
8139
8140                 TAILQ_REMOVE(&p->actions, action, node);
8141                 free(action->instructions);
8142                 free(action);
8143         }
8144 }
8145
8146 static uint32_t
8147 action_arg_src_mov_count(struct action *a,
8148                          uint32_t arg_id,
8149                          struct instruction *instructions,
8150                          struct instruction_data *instruction_data,
8151                          uint32_t n_instructions)
8152 {
8153         uint32_t offset, n_users = 0, i;
8154
8155         if (!a->st ||
8156             (arg_id >= a->st->n_fields) ||
8157             !instructions ||
8158             !instruction_data ||
8159             !n_instructions)
8160                 return 0;
8161
8162         offset = a->st->fields[arg_id].offset / 8;
8163
8164         for (i = 0; i < n_instructions; i++) {
8165                 struct instruction *instr = &instructions[i];
8166                 struct instruction_data *data = &instruction_data[i];
8167
8168                 if (data->invalid ||
8169                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
8170                     instr->mov.src.struct_id ||
8171                     (instr->mov.src.offset != offset))
8172                         continue;
8173
8174                 n_users++;
8175         }
8176
8177         return n_users;
8178 }
8179
8180 /*
8181  * Table.
8182  */
8183 static struct table_type *
8184 table_type_find(struct rte_swx_pipeline *p, const char *name)
8185 {
8186         struct table_type *elem;
8187
8188         TAILQ_FOREACH(elem, &p->table_types, node)
8189                 if (strcmp(elem->name, name) == 0)
8190                         return elem;
8191
8192         return NULL;
8193 }
8194
8195 static struct table_type *
8196 table_type_resolve(struct rte_swx_pipeline *p,
8197                    const char *recommended_type_name,
8198                    enum rte_swx_table_match_type match_type)
8199 {
8200         struct table_type *elem;
8201
8202         /* Only consider the recommended type if the match type is correct. */
8203         if (recommended_type_name)
8204                 TAILQ_FOREACH(elem, &p->table_types, node)
8205                         if (!strcmp(elem->name, recommended_type_name) &&
8206                             (elem->match_type == match_type))
8207                                 return elem;
8208
8209         /* Ignore the recommended type and get the first element with this match
8210          * type.
8211          */
8212         TAILQ_FOREACH(elem, &p->table_types, node)
8213                 if (elem->match_type == match_type)
8214                         return elem;
8215
8216         return NULL;
8217 }
8218
8219 static struct table *
8220 table_find(struct rte_swx_pipeline *p, const char *name)
8221 {
8222         struct table *elem;
8223
8224         TAILQ_FOREACH(elem, &p->tables, node)
8225                 if (strcmp(elem->name, name) == 0)
8226                         return elem;
8227
8228         return NULL;
8229 }
8230
8231 static struct table *
8232 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8233 {
8234         struct table *table = NULL;
8235
8236         TAILQ_FOREACH(table, &p->tables, node)
8237                 if (table->id == id)
8238                         return table;
8239
8240         return NULL;
8241 }
8242
8243 int
8244 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8245                                      const char *name,
8246                                      enum rte_swx_table_match_type match_type,
8247                                      struct rte_swx_table_ops *ops)
8248 {
8249         struct table_type *elem;
8250
8251         CHECK(p, EINVAL);
8252
8253         CHECK_NAME(name, EINVAL);
8254         CHECK(!table_type_find(p, name), EEXIST);
8255
8256         CHECK(ops, EINVAL);
8257         CHECK(ops->create, EINVAL);
8258         CHECK(ops->lkp, EINVAL);
8259         CHECK(ops->free, EINVAL);
8260
8261         /* Node allocation. */
8262         elem = calloc(1, sizeof(struct table_type));
8263         CHECK(elem, ENOMEM);
8264
8265         /* Node initialization. */
8266         strcpy(elem->name, name);
8267         elem->match_type = match_type;
8268         memcpy(&elem->ops, ops, sizeof(*ops));
8269
8270         /* Node add to tailq. */
8271         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8272
8273         return 0;
8274 }
8275
8276 static int
8277 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8278                          uint32_t n_fields,
8279                          enum rte_swx_table_match_type *match_type)
8280 {
8281         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
8282
8283         for (i = 0; i < n_fields; i++) {
8284                 struct rte_swx_match_field_params  *f = &fields[i];
8285
8286                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
8287                         n_fields_em++;
8288
8289                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
8290                         n_fields_lpm++;
8291         }
8292
8293         if ((n_fields_lpm > 1) ||
8294             (n_fields_lpm && (n_fields_em != n_fields - 1)))
8295                 return -EINVAL;
8296
8297         *match_type = (n_fields_em == n_fields) ?
8298                        RTE_SWX_TABLE_MATCH_EXACT :
8299                        RTE_SWX_TABLE_MATCH_WILDCARD;
8300
8301         return 0;
8302 }
8303
8304 static int
8305 table_match_fields_check(struct rte_swx_pipeline *p,
8306                          struct rte_swx_pipeline_table_params *params,
8307                          struct header **header)
8308 {
8309         struct header *h0 = NULL;
8310         struct field *hf, *mf;
8311         uint32_t *offset = NULL, i;
8312         int status = 0;
8313
8314         /* Return if no match fields. */
8315         if (!params->n_fields) {
8316                 if (params->fields) {
8317                         status = -EINVAL;
8318                         goto end;
8319                 }
8320
8321                 if (header)
8322                         *header = NULL;
8323
8324                 return 0;
8325         }
8326
8327         /* Memory allocation. */
8328         offset = calloc(params->n_fields, sizeof(uint32_t));
8329         if (!offset) {
8330                 status = -ENOMEM;
8331                 goto end;
8332         }
8333
8334         /* Check that all the match fields belong to either the same header or
8335          * to the meta-data.
8336          */
8337         hf = header_field_parse(p, params->fields[0].name, &h0);
8338         mf = metadata_field_parse(p, params->fields[0].name);
8339         if ((!hf && !mf) || (hf && hf->var_size)) {
8340                 status = -EINVAL;
8341                 goto end;
8342         }
8343
8344         offset[0] = h0 ? hf->offset : mf->offset;
8345
8346         for (i = 1; i < params->n_fields; i++)
8347                 if (h0) {
8348                         struct header *h;
8349
8350                         hf = header_field_parse(p, params->fields[i].name, &h);
8351                         if (!hf || (h->id != h0->id) || hf->var_size) {
8352                                 status = -EINVAL;
8353                                 goto end;
8354                         }
8355
8356                         offset[i] = hf->offset;
8357                 } else {
8358                         mf = metadata_field_parse(p, params->fields[i].name);
8359                         if (!mf) {
8360                                 status = -EINVAL;
8361                                 goto end;
8362                         }
8363
8364                         offset[i] = mf->offset;
8365                 }
8366
8367         /* Check that there are no duplicated match fields. */
8368         for (i = 0; i < params->n_fields; i++) {
8369                 uint32_t j;
8370
8371                 for (j = 0; j < i; j++)
8372                         if (offset[j] == offset[i]) {
8373                                 status = -EINVAL;
8374                                 goto end;
8375                         }
8376         }
8377
8378         /* Return. */
8379         if (header)
8380                 *header = h0;
8381
8382 end:
8383         free(offset);
8384         return status;
8385 }
8386
8387 int
8388 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8389                               const char *name,
8390                               struct rte_swx_pipeline_table_params *params,
8391                               const char *recommended_table_type_name,
8392                               const char *args,
8393                               uint32_t size)
8394 {
8395         struct table_type *type;
8396         struct table *t;
8397         struct action *default_action;
8398         struct header *header = NULL;
8399         uint32_t action_data_size_max = 0, i;
8400         int status = 0;
8401
8402         CHECK(p, EINVAL);
8403
8404         CHECK_NAME(name, EINVAL);
8405         CHECK(!table_find(p, name), EEXIST);
8406         CHECK(!selector_find(p, name), EEXIST);
8407         CHECK(!learner_find(p, name), EEXIST);
8408
8409         CHECK(params, EINVAL);
8410
8411         /* Match checks. */
8412         status = table_match_fields_check(p, params, &header);
8413         if (status)
8414                 return status;
8415
8416         /* Action checks. */
8417         CHECK(params->n_actions, EINVAL);
8418         CHECK(params->action_names, EINVAL);
8419         for (i = 0; i < params->n_actions; i++) {
8420                 const char *action_name = params->action_names[i];
8421                 struct action *a;
8422                 uint32_t action_data_size;
8423
8424                 CHECK_NAME(action_name, EINVAL);
8425
8426                 a = action_find(p, action_name);
8427                 CHECK(a, EINVAL);
8428                 CHECK(!action_does_learning(a), EINVAL);
8429
8430                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8431                 if (action_data_size > action_data_size_max)
8432                         action_data_size_max = action_data_size;
8433         }
8434
8435         CHECK_NAME(params->default_action_name, EINVAL);
8436         for (i = 0; i < p->n_actions; i++)
8437                 if (!strcmp(params->action_names[i],
8438                             params->default_action_name))
8439                         break;
8440         CHECK(i < params->n_actions, EINVAL);
8441         default_action = action_find(p, params->default_action_name);
8442         CHECK((default_action->st && params->default_action_data) ||
8443               !params->default_action_data, EINVAL);
8444
8445         /* Table type checks. */
8446         if (recommended_table_type_name)
8447                 CHECK_NAME(recommended_table_type_name, EINVAL);
8448
8449         if (params->n_fields) {
8450                 enum rte_swx_table_match_type match_type;
8451
8452                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8453                 if (status)
8454                         return status;
8455
8456                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8457                 CHECK(type, EINVAL);
8458         } else {
8459                 type = NULL;
8460         }
8461
8462         /* Memory allocation. */
8463         t = calloc(1, sizeof(struct table));
8464         CHECK(t, ENOMEM);
8465
8466         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8467         if (!t->fields) {
8468                 free(t);
8469                 CHECK(0, ENOMEM);
8470         }
8471
8472         t->actions = calloc(params->n_actions, sizeof(struct action *));
8473         if (!t->actions) {
8474                 free(t->fields);
8475                 free(t);
8476                 CHECK(0, ENOMEM);
8477         }
8478
8479         if (action_data_size_max) {
8480                 t->default_action_data = calloc(1, action_data_size_max);
8481                 if (!t->default_action_data) {
8482                         free(t->actions);
8483                         free(t->fields);
8484                         free(t);
8485                         CHECK(0, ENOMEM);
8486                 }
8487         }
8488
8489         /* Node initialization. */
8490         strcpy(t->name, name);
8491         if (args && args[0])
8492                 strcpy(t->args, args);
8493         t->type = type;
8494
8495         for (i = 0; i < params->n_fields; i++) {
8496                 struct rte_swx_match_field_params *field = &params->fields[i];
8497                 struct match_field *f = &t->fields[i];
8498
8499                 f->match_type = field->match_type;
8500                 f->field = header ?
8501                         header_field_parse(p, field->name, NULL) :
8502                         metadata_field_parse(p, field->name);
8503         }
8504         t->n_fields = params->n_fields;
8505         t->header = header;
8506
8507         for (i = 0; i < params->n_actions; i++)
8508                 t->actions[i] = action_find(p, params->action_names[i]);
8509         t->default_action = default_action;
8510         if (default_action->st)
8511                 memcpy(t->default_action_data,
8512                        params->default_action_data,
8513                        default_action->st->n_bits / 8);
8514         t->n_actions = params->n_actions;
8515         t->default_action_is_const = params->default_action_is_const;
8516         t->action_data_size_max = action_data_size_max;
8517
8518         t->size = size;
8519         t->id = p->n_tables;
8520
8521         /* Node add to tailq. */
8522         TAILQ_INSERT_TAIL(&p->tables, t, node);
8523         p->n_tables++;
8524
8525         return 0;
8526 }
8527
8528 static struct rte_swx_table_params *
8529 table_params_get(struct table *table)
8530 {
8531         struct rte_swx_table_params *params;
8532         struct field *first, *last;
8533         uint8_t *key_mask;
8534         uint32_t key_size, key_offset, action_data_size, i;
8535
8536         /* Memory allocation. */
8537         params = calloc(1, sizeof(struct rte_swx_table_params));
8538         if (!params)
8539                 return NULL;
8540
8541         /* Find first (smallest offset) and last (biggest offset) match fields. */
8542         first = table->fields[0].field;
8543         last = table->fields[0].field;
8544
8545         for (i = 0; i < table->n_fields; i++) {
8546                 struct field *f = table->fields[i].field;
8547
8548                 if (f->offset < first->offset)
8549                         first = f;
8550
8551                 if (f->offset > last->offset)
8552                         last = f;
8553         }
8554
8555         /* Key offset and size. */
8556         key_offset = first->offset / 8;
8557         key_size = (last->offset + last->n_bits - first->offset) / 8;
8558
8559         /* Memory allocation. */
8560         key_mask = calloc(1, key_size);
8561         if (!key_mask) {
8562                 free(params);
8563                 return NULL;
8564         }
8565
8566         /* Key mask. */
8567         for (i = 0; i < table->n_fields; i++) {
8568                 struct field *f = table->fields[i].field;
8569                 uint32_t start = (f->offset - first->offset) / 8;
8570                 size_t size = f->n_bits / 8;
8571
8572                 memset(&key_mask[start], 0xFF, size);
8573         }
8574
8575         /* Action data size. */
8576         action_data_size = 0;
8577         for (i = 0; i < table->n_actions; i++) {
8578                 struct action *action = table->actions[i];
8579                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8580
8581                 if (ads > action_data_size)
8582                         action_data_size = ads;
8583         }
8584
8585         /* Fill in. */
8586         params->match_type = table->type->match_type;
8587         params->key_size = key_size;
8588         params->key_offset = key_offset;
8589         params->key_mask0 = key_mask;
8590         params->action_data_size = action_data_size;
8591         params->n_keys_max = table->size;
8592
8593         return params;
8594 }
8595
8596 static void
8597 table_params_free(struct rte_swx_table_params *params)
8598 {
8599         if (!params)
8600                 return;
8601
8602         free(params->key_mask0);
8603         free(params);
8604 }
8605
8606 static int
8607 table_stub_lkp(void *table __rte_unused,
8608                void *mailbox __rte_unused,
8609                uint8_t **key __rte_unused,
8610                uint64_t *action_id __rte_unused,
8611                uint8_t **action_data __rte_unused,
8612                int *hit)
8613 {
8614         *hit = 0;
8615         return 1; /* DONE. */
8616 }
8617
8618 static int
8619 table_build(struct rte_swx_pipeline *p)
8620 {
8621         uint32_t i;
8622
8623         /* Per pipeline: table statistics. */
8624         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8625         CHECK(p->table_stats, ENOMEM);
8626
8627         for (i = 0; i < p->n_tables; i++) {
8628                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8629                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8630         }
8631
8632         /* Per thread: table runt-time. */
8633         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8634                 struct thread *t = &p->threads[i];
8635                 struct table *table;
8636
8637                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8638                 CHECK(t->tables, ENOMEM);
8639
8640                 TAILQ_FOREACH(table, &p->tables, node) {
8641                         struct table_runtime *r = &t->tables[table->id];
8642
8643                         if (table->type) {
8644                                 uint64_t size;
8645
8646                                 size = table->type->ops.mailbox_size_get();
8647
8648                                 /* r->func. */
8649                                 r->func = table->type->ops.lkp;
8650
8651                                 /* r->mailbox. */
8652                                 if (size) {
8653                                         r->mailbox = calloc(1, size);
8654                                         CHECK(r->mailbox, ENOMEM);
8655                                 }
8656
8657                                 /* r->key. */
8658                                 r->key = table->header ?
8659                                         &t->structs[table->header->struct_id] :
8660                                         &t->structs[p->metadata_struct_id];
8661                         } else {
8662                                 r->func = table_stub_lkp;
8663                         }
8664                 }
8665         }
8666
8667         return 0;
8668 }
8669
8670 static void
8671 table_build_free(struct rte_swx_pipeline *p)
8672 {
8673         uint32_t i;
8674
8675         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8676                 struct thread *t = &p->threads[i];
8677                 uint32_t j;
8678
8679                 if (!t->tables)
8680                         continue;
8681
8682                 for (j = 0; j < p->n_tables; j++) {
8683                         struct table_runtime *r = &t->tables[j];
8684
8685                         free(r->mailbox);
8686                 }
8687
8688                 free(t->tables);
8689                 t->tables = NULL;
8690         }
8691
8692         if (p->table_stats) {
8693                 for (i = 0; i < p->n_tables; i++)
8694                         free(p->table_stats[i].n_pkts_action);
8695
8696                 free(p->table_stats);
8697         }
8698 }
8699
8700 static void
8701 table_free(struct rte_swx_pipeline *p)
8702 {
8703         table_build_free(p);
8704
8705         /* Tables. */
8706         for ( ; ; ) {
8707                 struct table *elem;
8708
8709                 elem = TAILQ_FIRST(&p->tables);
8710                 if (!elem)
8711                         break;
8712
8713                 TAILQ_REMOVE(&p->tables, elem, node);
8714                 free(elem->fields);
8715                 free(elem->actions);
8716                 free(elem->default_action_data);
8717                 free(elem);
8718         }
8719
8720         /* Table types. */
8721         for ( ; ; ) {
8722                 struct table_type *elem;
8723
8724                 elem = TAILQ_FIRST(&p->table_types);
8725                 if (!elem)
8726                         break;
8727
8728                 TAILQ_REMOVE(&p->table_types, elem, node);
8729                 free(elem);
8730         }
8731 }
8732
8733 /*
8734  * Selector.
8735  */
8736 static struct selector *
8737 selector_find(struct rte_swx_pipeline *p, const char *name)
8738 {
8739         struct selector *s;
8740
8741         TAILQ_FOREACH(s, &p->selectors, node)
8742                 if (strcmp(s->name, name) == 0)
8743                         return s;
8744
8745         return NULL;
8746 }
8747
8748 static struct selector *
8749 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8750 {
8751         struct selector *s = NULL;
8752
8753         TAILQ_FOREACH(s, &p->selectors, node)
8754                 if (s->id == id)
8755                         return s;
8756
8757         return NULL;
8758 }
8759
8760 static int
8761 selector_fields_check(struct rte_swx_pipeline *p,
8762                       struct rte_swx_pipeline_selector_params *params,
8763                       struct header **header)
8764 {
8765         struct header *h0 = NULL;
8766         struct field *hf, *mf;
8767         uint32_t i;
8768
8769         /* Return if no selector fields. */
8770         if (!params->n_selector_fields || !params->selector_field_names)
8771                 return -EINVAL;
8772
8773         /* Check that all the selector fields either belong to the same header
8774          * or are all meta-data fields.
8775          */
8776         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8777         mf = metadata_field_parse(p, params->selector_field_names[0]);
8778         if (!hf && !mf)
8779                 return -EINVAL;
8780
8781         for (i = 1; i < params->n_selector_fields; i++)
8782                 if (h0) {
8783                         struct header *h;
8784
8785                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8786                         if (!hf || (h->id != h0->id))
8787                                 return -EINVAL;
8788                 } else {
8789                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8790                         if (!mf)
8791                                 return -EINVAL;
8792                 }
8793
8794         /* Check that there are no duplicated match fields. */
8795         for (i = 0; i < params->n_selector_fields; i++) {
8796                 const char *field_name = params->selector_field_names[i];
8797                 uint32_t j;
8798
8799                 for (j = i + 1; j < params->n_selector_fields; j++)
8800                         if (!strcmp(params->selector_field_names[j], field_name))
8801                                 return -EINVAL;
8802         }
8803
8804         /* Return. */
8805         if (header)
8806                 *header = h0;
8807
8808         return 0;
8809 }
8810
8811 int
8812 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8813                                  const char *name,
8814                                  struct rte_swx_pipeline_selector_params *params)
8815 {
8816         struct selector *s;
8817         struct header *selector_header = NULL;
8818         struct field *group_id_field, *member_id_field;
8819         uint32_t i;
8820         int status = 0;
8821
8822         CHECK(p, EINVAL);
8823
8824         CHECK_NAME(name, EINVAL);
8825         CHECK(!table_find(p, name), EEXIST);
8826         CHECK(!selector_find(p, name), EEXIST);
8827         CHECK(!learner_find(p, name), EEXIST);
8828
8829         CHECK(params, EINVAL);
8830
8831         CHECK_NAME(params->group_id_field_name, EINVAL);
8832         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8833         CHECK(group_id_field, EINVAL);
8834
8835         for (i = 0; i < params->n_selector_fields; i++) {
8836                 const char *field_name = params->selector_field_names[i];
8837
8838                 CHECK_NAME(field_name, EINVAL);
8839         }
8840         status = selector_fields_check(p, params, &selector_header);
8841         if (status)
8842                 return status;
8843
8844         CHECK_NAME(params->member_id_field_name, EINVAL);
8845         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8846         CHECK(member_id_field, EINVAL);
8847
8848         CHECK(params->n_groups_max, EINVAL);
8849
8850         CHECK(params->n_members_per_group_max, EINVAL);
8851
8852         /* Memory allocation. */
8853         s = calloc(1, sizeof(struct selector));
8854         if (!s) {
8855                 status = -ENOMEM;
8856                 goto error;
8857         }
8858
8859         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8860         if (!s->selector_fields) {
8861                 status = -ENOMEM;
8862                 goto error;
8863         }
8864
8865         /* Node initialization. */
8866         strcpy(s->name, name);
8867
8868         s->group_id_field = group_id_field;
8869
8870         for (i = 0; i < params->n_selector_fields; i++) {
8871                 const char *field_name = params->selector_field_names[i];
8872
8873                 s->selector_fields[i] = selector_header ?
8874                         header_field_parse(p, field_name, NULL) :
8875                         metadata_field_parse(p, field_name);
8876         }
8877
8878         s->n_selector_fields = params->n_selector_fields;
8879
8880         s->selector_header = selector_header;
8881
8882         s->member_id_field = member_id_field;
8883
8884         s->n_groups_max = params->n_groups_max;
8885
8886         s->n_members_per_group_max = params->n_members_per_group_max;
8887
8888         s->id = p->n_selectors;
8889
8890         /* Node add to tailq. */
8891         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8892         p->n_selectors++;
8893
8894         return 0;
8895
8896 error:
8897         if (!s)
8898                 return status;
8899
8900         free(s->selector_fields);
8901
8902         free(s);
8903
8904         return status;
8905 }
8906
8907 static void
8908 selector_params_free(struct rte_swx_table_selector_params *params)
8909 {
8910         if (!params)
8911                 return;
8912
8913         free(params->selector_mask);
8914
8915         free(params);
8916 }
8917
8918 static struct rte_swx_table_selector_params *
8919 selector_table_params_get(struct selector *s)
8920 {
8921         struct rte_swx_table_selector_params *params = NULL;
8922         struct field *first, *last;
8923         uint32_t i;
8924
8925         /* Memory allocation. */
8926         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8927         if (!params)
8928                 goto error;
8929
8930         /* Group ID. */
8931         params->group_id_offset = s->group_id_field->offset / 8;
8932
8933         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8934         first = s->selector_fields[0];
8935         last = s->selector_fields[0];
8936
8937         for (i = 0; i < s->n_selector_fields; i++) {
8938                 struct field *f = s->selector_fields[i];
8939
8940                 if (f->offset < first->offset)
8941                         first = f;
8942
8943                 if (f->offset > last->offset)
8944                         last = f;
8945         }
8946
8947         /* Selector offset and size. */
8948         params->selector_offset = first->offset / 8;
8949         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8950
8951         /* Memory allocation. */
8952         params->selector_mask = calloc(1, params->selector_size);
8953         if (!params->selector_mask)
8954                 goto error;
8955
8956         /* Selector mask. */
8957         for (i = 0; i < s->n_selector_fields; i++) {
8958                 struct field *f = s->selector_fields[i];
8959                 uint32_t start = (f->offset - first->offset) / 8;
8960                 size_t size = f->n_bits / 8;
8961
8962                 memset(&params->selector_mask[start], 0xFF, size);
8963         }
8964
8965         /* Member ID. */
8966         params->member_id_offset = s->member_id_field->offset / 8;
8967
8968         /* Maximum number of groups. */
8969         params->n_groups_max = s->n_groups_max;
8970
8971         /* Maximum number of members per group. */
8972         params->n_members_per_group_max = s->n_members_per_group_max;
8973
8974         return params;
8975
8976 error:
8977         selector_params_free(params);
8978         return NULL;
8979 }
8980
8981 static void
8982 selector_build_free(struct rte_swx_pipeline *p)
8983 {
8984         uint32_t i;
8985
8986         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8987                 struct thread *t = &p->threads[i];
8988                 uint32_t j;
8989
8990                 if (!t->selectors)
8991                         continue;
8992
8993                 for (j = 0; j < p->n_selectors; j++) {
8994                         struct selector_runtime *r = &t->selectors[j];
8995
8996                         free(r->mailbox);
8997                 }
8998
8999                 free(t->selectors);
9000                 t->selectors = NULL;
9001         }
9002
9003         free(p->selector_stats);
9004         p->selector_stats = NULL;
9005 }
9006
9007 static int
9008 selector_build(struct rte_swx_pipeline *p)
9009 {
9010         uint32_t i;
9011         int status = 0;
9012
9013         /* Per pipeline: selector statistics. */
9014         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
9015         if (!p->selector_stats) {
9016                 status = -ENOMEM;
9017                 goto error;
9018         }
9019
9020         /* Per thread: selector run-time. */
9021         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9022                 struct thread *t = &p->threads[i];
9023                 struct selector *s;
9024
9025                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
9026                 if (!t->selectors) {
9027                         status = -ENOMEM;
9028                         goto error;
9029                 }
9030
9031                 TAILQ_FOREACH(s, &p->selectors, node) {
9032                         struct selector_runtime *r = &t->selectors[s->id];
9033                         uint64_t size;
9034
9035                         /* r->mailbox. */
9036                         size = rte_swx_table_selector_mailbox_size_get();
9037                         if (size) {
9038                                 r->mailbox = calloc(1, size);
9039                                 if (!r->mailbox) {
9040                                         status = -ENOMEM;
9041                                         goto error;
9042                                 }
9043                         }
9044
9045                         /* r->group_id_buffer. */
9046                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
9047
9048                         /* r->selector_buffer. */
9049                         r->selector_buffer = s->selector_header ?
9050                                 &t->structs[s->selector_header->struct_id] :
9051                                 &t->structs[p->metadata_struct_id];
9052
9053                         /* r->member_id_buffer. */
9054                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
9055                 }
9056         }
9057
9058         return 0;
9059
9060 error:
9061         selector_build_free(p);
9062         return status;
9063 }
9064
9065 static void
9066 selector_free(struct rte_swx_pipeline *p)
9067 {
9068         selector_build_free(p);
9069
9070         /* Selector tables. */
9071         for ( ; ; ) {
9072                 struct selector *elem;
9073
9074                 elem = TAILQ_FIRST(&p->selectors);
9075                 if (!elem)
9076                         break;
9077
9078                 TAILQ_REMOVE(&p->selectors, elem, node);
9079                 free(elem->selector_fields);
9080                 free(elem);
9081         }
9082 }
9083
9084 /*
9085  * Learner table.
9086  */
9087 static struct learner *
9088 learner_find(struct rte_swx_pipeline *p, const char *name)
9089 {
9090         struct learner *l;
9091
9092         TAILQ_FOREACH(l, &p->learners, node)
9093                 if (!strcmp(l->name, name))
9094                         return l;
9095
9096         return NULL;
9097 }
9098
9099 static struct learner *
9100 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9101 {
9102         struct learner *l = NULL;
9103
9104         TAILQ_FOREACH(l, &p->learners, node)
9105                 if (l->id == id)
9106                         return l;
9107
9108         return NULL;
9109 }
9110
9111 static int
9112 learner_match_fields_check(struct rte_swx_pipeline *p,
9113                            struct rte_swx_pipeline_learner_params *params,
9114                            struct header **header)
9115 {
9116         struct header *h0 = NULL;
9117         struct field *hf, *mf;
9118         uint32_t i;
9119
9120         /* Return if no match fields. */
9121         if (!params->n_fields || !params->field_names)
9122                 return -EINVAL;
9123
9124         /* Check that all the match fields either belong to the same header
9125          * or are all meta-data fields.
9126          */
9127         hf = header_field_parse(p, params->field_names[0], &h0);
9128         mf = metadata_field_parse(p, params->field_names[0]);
9129         if (!hf && !mf)
9130                 return -EINVAL;
9131
9132         for (i = 1; i < params->n_fields; i++)
9133                 if (h0) {
9134                         struct header *h;
9135
9136                         hf = header_field_parse(p, params->field_names[i], &h);
9137                         if (!hf || (h->id != h0->id))
9138                                 return -EINVAL;
9139                 } else {
9140                         mf = metadata_field_parse(p, params->field_names[i]);
9141                         if (!mf)
9142                                 return -EINVAL;
9143                 }
9144
9145         /* Check that there are no duplicated match fields. */
9146         for (i = 0; i < params->n_fields; i++) {
9147                 const char *field_name = params->field_names[i];
9148                 uint32_t j;
9149
9150                 for (j = i + 1; j < params->n_fields; j++)
9151                         if (!strcmp(params->field_names[j], field_name))
9152                                 return -EINVAL;
9153         }
9154
9155         /* Return. */
9156         if (header)
9157                 *header = h0;
9158
9159         return 0;
9160 }
9161
9162 static int
9163 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
9164 {
9165         struct struct_type *mst = p->metadata_st, *ast = a->st;
9166         struct field *mf, *af;
9167         uint32_t mf_pos, i;
9168
9169         if (!ast) {
9170                 if (mf_name)
9171                         return -EINVAL;
9172
9173                 return 0;
9174         }
9175
9176         /* Check that mf_name is the name of a valid meta-data field. */
9177         CHECK_NAME(mf_name, EINVAL);
9178         mf = metadata_field_parse(p, mf_name);
9179         CHECK(mf, EINVAL);
9180
9181         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
9182          * all the action arguments.
9183          */
9184         mf_pos = mf - mst->fields;
9185         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
9186
9187         /* Check that the size of each of the identified meta-data fields matches exactly the size
9188          * of the corresponding action argument.
9189          */
9190         for (i = 0; i < ast->n_fields; i++) {
9191                 mf = &mst->fields[mf_pos + i];
9192                 af = &ast->fields[i];
9193
9194                 CHECK(mf->n_bits == af->n_bits, EINVAL);
9195         }
9196
9197         return 0;
9198 }
9199
9200 static int
9201 learner_action_learning_check(struct rte_swx_pipeline *p,
9202                               struct action *action,
9203                               const char **action_names,
9204                               uint32_t n_actions)
9205 {
9206         uint32_t i;
9207
9208         /* For each "learn" instruction of the current action, check that the learned action (i.e.
9209          * the action passed as argument to the "learn" instruction) is also enabled for the
9210          * current learner table.
9211          */
9212         for (i = 0; i < action->n_instructions; i++) {
9213                 struct instruction *instr = &action->instructions[i];
9214                 uint32_t found = 0, j;
9215
9216                 if (instr->type != INSTR_LEARNER_LEARN)
9217                         continue;
9218
9219                 for (j = 0; j < n_actions; j++) {
9220                         struct action *a;
9221
9222                         a = action_find(p, action_names[j]);
9223                         if (!a)
9224                                 return -EINVAL;
9225
9226                         if (a->id == instr->learn.action_id)
9227                                 found = 1;
9228                 }
9229
9230                 if (!found)
9231                         return -EINVAL;
9232         }
9233
9234         return 0;
9235 }
9236
9237 int
9238 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
9239                               const char *name,
9240                               struct rte_swx_pipeline_learner_params *params,
9241                               uint32_t size,
9242                               uint32_t timeout)
9243 {
9244         struct learner *l = NULL;
9245         struct action *default_action;
9246         struct header *header = NULL;
9247         uint32_t action_data_size_max = 0, i;
9248         int status = 0;
9249
9250         CHECK(p, EINVAL);
9251
9252         CHECK_NAME(name, EINVAL);
9253         CHECK(!table_find(p, name), EEXIST);
9254         CHECK(!selector_find(p, name), EEXIST);
9255         CHECK(!learner_find(p, name), EEXIST);
9256
9257         CHECK(params, EINVAL);
9258
9259         /* Match checks. */
9260         status = learner_match_fields_check(p, params, &header);
9261         if (status)
9262                 return status;
9263
9264         /* Action checks. */
9265         CHECK(params->n_actions, EINVAL);
9266
9267         CHECK(params->action_names, EINVAL);
9268         for (i = 0; i < params->n_actions; i++) {
9269                 const char *action_name = params->action_names[i];
9270                 const char *action_field_name = params->action_field_names[i];
9271                 struct action *a;
9272                 uint32_t action_data_size;
9273
9274                 CHECK_NAME(action_name, EINVAL);
9275
9276                 a = action_find(p, action_name);
9277                 CHECK(a, EINVAL);
9278
9279                 status = learner_action_args_check(p, a, action_field_name);
9280                 if (status)
9281                         return status;
9282
9283                 status = learner_action_learning_check(p,
9284                                                        a,
9285                                                        params->action_names,
9286                                                        params->n_actions);
9287                 if (status)
9288                         return status;
9289
9290                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9291                 if (action_data_size > action_data_size_max)
9292                         action_data_size_max = action_data_size;
9293         }
9294
9295         CHECK_NAME(params->default_action_name, EINVAL);
9296         for (i = 0; i < p->n_actions; i++)
9297                 if (!strcmp(params->action_names[i],
9298                             params->default_action_name))
9299                         break;
9300         CHECK(i < params->n_actions, EINVAL);
9301
9302         default_action = action_find(p, params->default_action_name);
9303         CHECK((default_action->st && params->default_action_data) ||
9304               !params->default_action_data, EINVAL);
9305
9306         /* Any other checks. */
9307         CHECK(size, EINVAL);
9308         CHECK(timeout, EINVAL);
9309
9310         /* Memory allocation. */
9311         l = calloc(1, sizeof(struct learner));
9312         if (!l)
9313                 goto nomem;
9314
9315         l->fields = calloc(params->n_fields, sizeof(struct field *));
9316         if (!l->fields)
9317                 goto nomem;
9318
9319         l->actions = calloc(params->n_actions, sizeof(struct action *));
9320         if (!l->actions)
9321                 goto nomem;
9322
9323         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9324         if (!l->action_arg)
9325                 goto nomem;
9326
9327         if (action_data_size_max) {
9328                 l->default_action_data = calloc(1, action_data_size_max);
9329                 if (!l->default_action_data)
9330                         goto nomem;
9331         }
9332
9333         /* Node initialization. */
9334         strcpy(l->name, name);
9335
9336         for (i = 0; i < params->n_fields; i++) {
9337                 const char *field_name = params->field_names[i];
9338
9339                 l->fields[i] = header ?
9340                         header_field_parse(p, field_name, NULL) :
9341                         metadata_field_parse(p, field_name);
9342         }
9343
9344         l->n_fields = params->n_fields;
9345
9346         l->header = header;
9347
9348         for (i = 0; i < params->n_actions; i++) {
9349                 const char *mf_name = params->action_field_names[i];
9350
9351                 l->actions[i] = action_find(p, params->action_names[i]);
9352
9353                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9354         }
9355
9356         l->default_action = default_action;
9357
9358         if (default_action->st)
9359                 memcpy(l->default_action_data,
9360                        params->default_action_data,
9361                        default_action->st->n_bits / 8);
9362
9363         l->n_actions = params->n_actions;
9364
9365         l->default_action_is_const = params->default_action_is_const;
9366
9367         l->action_data_size_max = action_data_size_max;
9368
9369         l->size = size;
9370
9371         l->timeout = timeout;
9372
9373         l->id = p->n_learners;
9374
9375         /* Node add to tailq. */
9376         TAILQ_INSERT_TAIL(&p->learners, l, node);
9377         p->n_learners++;
9378
9379         return 0;
9380
9381 nomem:
9382         if (!l)
9383                 return -ENOMEM;
9384
9385         free(l->action_arg);
9386         free(l->actions);
9387         free(l->fields);
9388         free(l);
9389
9390         return -ENOMEM;
9391 }
9392
9393 static void
9394 learner_params_free(struct rte_swx_table_learner_params *params)
9395 {
9396         if (!params)
9397                 return;
9398
9399         free(params->key_mask0);
9400
9401         free(params);
9402 }
9403
9404 static struct rte_swx_table_learner_params *
9405 learner_params_get(struct learner *l)
9406 {
9407         struct rte_swx_table_learner_params *params = NULL;
9408         struct field *first, *last;
9409         uint32_t i;
9410
9411         /* Memory allocation. */
9412         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9413         if (!params)
9414                 goto error;
9415
9416         /* Find first (smallest offset) and last (biggest offset) match fields. */
9417         first = l->fields[0];
9418         last = l->fields[0];
9419
9420         for (i = 0; i < l->n_fields; i++) {
9421                 struct field *f = l->fields[i];
9422
9423                 if (f->offset < first->offset)
9424                         first = f;
9425
9426                 if (f->offset > last->offset)
9427                         last = f;
9428         }
9429
9430         /* Key offset and size. */
9431         params->key_offset = first->offset / 8;
9432         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9433
9434         /* Memory allocation. */
9435         params->key_mask0 = calloc(1, params->key_size);
9436         if (!params->key_mask0)
9437                 goto error;
9438
9439         /* Key mask. */
9440         for (i = 0; i < l->n_fields; i++) {
9441                 struct field *f = l->fields[i];
9442                 uint32_t start = (f->offset - first->offset) / 8;
9443                 size_t size = f->n_bits / 8;
9444
9445                 memset(&params->key_mask0[start], 0xFF, size);
9446         }
9447
9448         /* Action data size. */
9449         params->action_data_size = l->action_data_size_max;
9450
9451         /* Maximum number of keys. */
9452         params->n_keys_max = l->size;
9453
9454         /* Timeout. */
9455         params->key_timeout = l->timeout;
9456
9457         return params;
9458
9459 error:
9460         learner_params_free(params);
9461         return NULL;
9462 }
9463
9464 static void
9465 learner_build_free(struct rte_swx_pipeline *p)
9466 {
9467         uint32_t i;
9468
9469         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9470                 struct thread *t = &p->threads[i];
9471                 uint32_t j;
9472
9473                 if (!t->learners)
9474                         continue;
9475
9476                 for (j = 0; j < p->n_learners; j++) {
9477                         struct learner_runtime *r = &t->learners[j];
9478
9479                         free(r->mailbox);
9480                         free(r->action_data);
9481                 }
9482
9483                 free(t->learners);
9484                 t->learners = NULL;
9485         }
9486
9487         if (p->learner_stats) {
9488                 for (i = 0; i < p->n_learners; i++)
9489                         free(p->learner_stats[i].n_pkts_action);
9490
9491                 free(p->learner_stats);
9492         }
9493 }
9494
9495 static int
9496 learner_build(struct rte_swx_pipeline *p)
9497 {
9498         uint32_t i;
9499         int status = 0;
9500
9501         /* Per pipeline: learner statistics. */
9502         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9503         CHECK(p->learner_stats, ENOMEM);
9504
9505         for (i = 0; i < p->n_learners; i++) {
9506                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9507                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9508         }
9509
9510         /* Per thread: learner run-time. */
9511         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9512                 struct thread *t = &p->threads[i];
9513                 struct learner *l;
9514
9515                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9516                 if (!t->learners) {
9517                         status = -ENOMEM;
9518                         goto error;
9519                 }
9520
9521                 TAILQ_FOREACH(l, &p->learners, node) {
9522                         struct learner_runtime *r = &t->learners[l->id];
9523                         uint64_t size;
9524                         uint32_t j;
9525
9526                         /* r->mailbox. */
9527                         size = rte_swx_table_learner_mailbox_size_get();
9528                         if (size) {
9529                                 r->mailbox = calloc(1, size);
9530                                 if (!r->mailbox) {
9531                                         status = -ENOMEM;
9532                                         goto error;
9533                                 }
9534                         }
9535
9536                         /* r->key. */
9537                         r->key = l->header ?
9538                                 &t->structs[l->header->struct_id] :
9539                                 &t->structs[p->metadata_struct_id];
9540
9541                         /* r->action_data. */
9542                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9543                         if (!r->action_data) {
9544                                 status = -ENOMEM;
9545                                 goto error;
9546                         }
9547
9548                         for (j = 0; j < l->n_actions; j++) {
9549                                 struct action *a = l->actions[j];
9550                                 struct field *mf = l->action_arg[j];
9551                                 uint8_t *m = t->structs[p->metadata_struct_id];
9552
9553                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9554                         }
9555                 }
9556         }
9557
9558         return 0;
9559
9560 error:
9561         learner_build_free(p);
9562         return status;
9563 }
9564
9565 static void
9566 learner_free(struct rte_swx_pipeline *p)
9567 {
9568         learner_build_free(p);
9569
9570         /* Learner tables. */
9571         for ( ; ; ) {
9572                 struct learner *l;
9573
9574                 l = TAILQ_FIRST(&p->learners);
9575                 if (!l)
9576                         break;
9577
9578                 TAILQ_REMOVE(&p->learners, l, node);
9579                 free(l->fields);
9580                 free(l->actions);
9581                 free(l->action_arg);
9582                 free(l->default_action_data);
9583                 free(l);
9584         }
9585 }
9586
9587 /*
9588  * Table state.
9589  */
9590 static int
9591 table_state_build(struct rte_swx_pipeline *p)
9592 {
9593         struct table *table;
9594         struct selector *s;
9595         struct learner *l;
9596
9597         p->table_state = calloc(p->n_tables + p->n_selectors,
9598                                 sizeof(struct rte_swx_table_state));
9599         CHECK(p->table_state, ENOMEM);
9600
9601         TAILQ_FOREACH(table, &p->tables, node) {
9602                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9603
9604                 if (table->type) {
9605                         struct rte_swx_table_params *params;
9606
9607                         /* ts->obj. */
9608                         params = table_params_get(table);
9609                         CHECK(params, ENOMEM);
9610
9611                         ts->obj = table->type->ops.create(params,
9612                                 NULL,
9613                                 table->args,
9614                                 p->numa_node);
9615
9616                         table_params_free(params);
9617                         CHECK(ts->obj, ENODEV);
9618                 }
9619
9620                 /* ts->default_action_data. */
9621                 if (table->action_data_size_max) {
9622                         ts->default_action_data =
9623                                 malloc(table->action_data_size_max);
9624                         CHECK(ts->default_action_data, ENOMEM);
9625
9626                         memcpy(ts->default_action_data,
9627                                table->default_action_data,
9628                                table->action_data_size_max);
9629                 }
9630
9631                 /* ts->default_action_id. */
9632                 ts->default_action_id = table->default_action->id;
9633         }
9634
9635         TAILQ_FOREACH(s, &p->selectors, node) {
9636                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9637                 struct rte_swx_table_selector_params *params;
9638
9639                 /* ts->obj. */
9640                 params = selector_table_params_get(s);
9641                 CHECK(params, ENOMEM);
9642
9643                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9644
9645                 selector_params_free(params);
9646                 CHECK(ts->obj, ENODEV);
9647         }
9648
9649         TAILQ_FOREACH(l, &p->learners, node) {
9650                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9651                         p->n_selectors + l->id];
9652                 struct rte_swx_table_learner_params *params;
9653
9654                 /* ts->obj. */
9655                 params = learner_params_get(l);
9656                 CHECK(params, ENOMEM);
9657
9658                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9659                 learner_params_free(params);
9660                 CHECK(ts->obj, ENODEV);
9661
9662                 /* ts->default_action_data. */
9663                 if (l->action_data_size_max) {
9664                         ts->default_action_data = malloc(l->action_data_size_max);
9665                         CHECK(ts->default_action_data, ENOMEM);
9666
9667                         memcpy(ts->default_action_data,
9668                                l->default_action_data,
9669                                l->action_data_size_max);
9670                 }
9671
9672                 /* ts->default_action_id. */
9673                 ts->default_action_id = l->default_action->id;
9674         }
9675
9676         return 0;
9677 }
9678
9679 static void
9680 table_state_build_free(struct rte_swx_pipeline *p)
9681 {
9682         uint32_t i;
9683
9684         if (!p->table_state)
9685                 return;
9686
9687         for (i = 0; i < p->n_tables; i++) {
9688                 struct rte_swx_table_state *ts = &p->table_state[i];
9689                 struct table *table = table_find_by_id(p, i);
9690
9691                 /* ts->obj. */
9692                 if (table->type && ts->obj)
9693                         table->type->ops.free(ts->obj);
9694
9695                 /* ts->default_action_data. */
9696                 free(ts->default_action_data);
9697         }
9698
9699         for (i = 0; i < p->n_selectors; i++) {
9700                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9701
9702                 /* ts->obj. */
9703                 if (ts->obj)
9704                         rte_swx_table_selector_free(ts->obj);
9705         }
9706
9707         for (i = 0; i < p->n_learners; i++) {
9708                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9709
9710                 /* ts->obj. */
9711                 if (ts->obj)
9712                         rte_swx_table_learner_free(ts->obj);
9713
9714                 /* ts->default_action_data. */
9715                 free(ts->default_action_data);
9716         }
9717
9718         free(p->table_state);
9719         p->table_state = NULL;
9720 }
9721
9722 static void
9723 table_state_free(struct rte_swx_pipeline *p)
9724 {
9725         table_state_build_free(p);
9726 }
9727
9728 /*
9729  * Register array.
9730  */
9731 static struct regarray *
9732 regarray_find(struct rte_swx_pipeline *p, const char *name)
9733 {
9734         struct regarray *elem;
9735
9736         TAILQ_FOREACH(elem, &p->regarrays, node)
9737                 if (!strcmp(elem->name, name))
9738                         return elem;
9739
9740         return NULL;
9741 }
9742
9743 static struct regarray *
9744 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9745 {
9746         struct regarray *elem = NULL;
9747
9748         TAILQ_FOREACH(elem, &p->regarrays, node)
9749                 if (elem->id == id)
9750                         return elem;
9751
9752         return NULL;
9753 }
9754
9755 int
9756 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9757                               const char *name,
9758                               uint32_t size,
9759                               uint64_t init_val)
9760 {
9761         struct regarray *r;
9762
9763         CHECK(p, EINVAL);
9764
9765         CHECK_NAME(name, EINVAL);
9766         CHECK(!regarray_find(p, name), EEXIST);
9767
9768         CHECK(size, EINVAL);
9769         size = rte_align32pow2(size);
9770
9771         /* Memory allocation. */
9772         r = calloc(1, sizeof(struct regarray));
9773         CHECK(r, ENOMEM);
9774
9775         /* Node initialization. */
9776         strcpy(r->name, name);
9777         r->init_val = init_val;
9778         r->size = size;
9779         r->id = p->n_regarrays;
9780
9781         /* Node add to tailq. */
9782         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9783         p->n_regarrays++;
9784
9785         return 0;
9786 }
9787
9788 static int
9789 regarray_build(struct rte_swx_pipeline *p)
9790 {
9791         struct regarray *regarray;
9792
9793         if (!p->n_regarrays)
9794                 return 0;
9795
9796         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9797         CHECK(p->regarray_runtime, ENOMEM);
9798
9799         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9800                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9801                 uint32_t i;
9802
9803                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9804                                          RTE_CACHE_LINE_SIZE,
9805                                          p->numa_node);
9806                 CHECK(r->regarray, ENOMEM);
9807
9808                 if (regarray->init_val)
9809                         for (i = 0; i < regarray->size; i++)
9810                                 r->regarray[i] = regarray->init_val;
9811
9812                 r->size_mask = regarray->size - 1;
9813         }
9814
9815         return 0;
9816 }
9817
9818 static void
9819 regarray_build_free(struct rte_swx_pipeline *p)
9820 {
9821         uint32_t i;
9822
9823         if (!p->regarray_runtime)
9824                 return;
9825
9826         for (i = 0; i < p->n_regarrays; i++) {
9827                 struct regarray *regarray = regarray_find_by_id(p, i);
9828                 struct regarray_runtime *r = &p->regarray_runtime[i];
9829
9830                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9831         }
9832
9833         free(p->regarray_runtime);
9834         p->regarray_runtime = NULL;
9835 }
9836
9837 static void
9838 regarray_free(struct rte_swx_pipeline *p)
9839 {
9840         regarray_build_free(p);
9841
9842         for ( ; ; ) {
9843                 struct regarray *elem;
9844
9845                 elem = TAILQ_FIRST(&p->regarrays);
9846                 if (!elem)
9847                         break;
9848
9849                 TAILQ_REMOVE(&p->regarrays, elem, node);
9850                 free(elem);
9851         }
9852 }
9853
9854 /*
9855  * Meter array.
9856  */
9857 static struct meter_profile *
9858 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9859 {
9860         struct meter_profile *elem;
9861
9862         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9863                 if (!strcmp(elem->name, name))
9864                         return elem;
9865
9866         return NULL;
9867 }
9868
9869 static struct metarray *
9870 metarray_find(struct rte_swx_pipeline *p, const char *name)
9871 {
9872         struct metarray *elem;
9873
9874         TAILQ_FOREACH(elem, &p->metarrays, node)
9875                 if (!strcmp(elem->name, name))
9876                         return elem;
9877
9878         return NULL;
9879 }
9880
9881 static struct metarray *
9882 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9883 {
9884         struct metarray *elem = NULL;
9885
9886         TAILQ_FOREACH(elem, &p->metarrays, node)
9887                 if (elem->id == id)
9888                         return elem;
9889
9890         return NULL;
9891 }
9892
9893 int
9894 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9895                                  const char *name,
9896                                  uint32_t size)
9897 {
9898         struct metarray *m;
9899
9900         CHECK(p, EINVAL);
9901
9902         CHECK_NAME(name, EINVAL);
9903         CHECK(!metarray_find(p, name), EEXIST);
9904
9905         CHECK(size, EINVAL);
9906         size = rte_align32pow2(size);
9907
9908         /* Memory allocation. */
9909         m = calloc(1, sizeof(struct metarray));
9910         CHECK(m, ENOMEM);
9911
9912         /* Node initialization. */
9913         strcpy(m->name, name);
9914         m->size = size;
9915         m->id = p->n_metarrays;
9916
9917         /* Node add to tailq. */
9918         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9919         p->n_metarrays++;
9920
9921         return 0;
9922 }
9923
9924 struct meter_profile meter_profile_default = {
9925         .node = {0},
9926         .name = "",
9927         .params = {0},
9928
9929         .profile = {
9930                 .cbs = 10000,
9931                 .pbs = 10000,
9932                 .cir_period = 1,
9933                 .cir_bytes_per_period = 1,
9934                 .pir_period = 1,
9935                 .pir_bytes_per_period = 1,
9936         },
9937
9938         .n_users = 0,
9939 };
9940
9941 static void
9942 meter_init(struct meter *m)
9943 {
9944         memset(m, 0, sizeof(struct meter));
9945         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9946         m->profile = &meter_profile_default;
9947         m->color_mask = RTE_COLOR_GREEN;
9948
9949         meter_profile_default.n_users++;
9950 }
9951
9952 static int
9953 metarray_build(struct rte_swx_pipeline *p)
9954 {
9955         struct metarray *m;
9956
9957         if (!p->n_metarrays)
9958                 return 0;
9959
9960         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9961         CHECK(p->metarray_runtime, ENOMEM);
9962
9963         TAILQ_FOREACH(m, &p->metarrays, node) {
9964                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9965                 uint32_t i;
9966
9967                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9968                                          RTE_CACHE_LINE_SIZE,
9969                                          p->numa_node);
9970                 CHECK(r->metarray, ENOMEM);
9971
9972                 for (i = 0; i < m->size; i++)
9973                         meter_init(&r->metarray[i]);
9974
9975                 r->size_mask = m->size - 1;
9976         }
9977
9978         return 0;
9979 }
9980
9981 static void
9982 metarray_build_free(struct rte_swx_pipeline *p)
9983 {
9984         uint32_t i;
9985
9986         if (!p->metarray_runtime)
9987                 return;
9988
9989         for (i = 0; i < p->n_metarrays; i++) {
9990                 struct metarray *m = metarray_find_by_id(p, i);
9991                 struct metarray_runtime *r = &p->metarray_runtime[i];
9992
9993                 env_free(r->metarray, m->size * sizeof(struct meter));
9994         }
9995
9996         free(p->metarray_runtime);
9997         p->metarray_runtime = NULL;
9998 }
9999
10000 static void
10001 metarray_free(struct rte_swx_pipeline *p)
10002 {
10003         metarray_build_free(p);
10004
10005         /* Meter arrays. */
10006         for ( ; ; ) {
10007                 struct metarray *elem;
10008
10009                 elem = TAILQ_FIRST(&p->metarrays);
10010                 if (!elem)
10011                         break;
10012
10013                 TAILQ_REMOVE(&p->metarrays, elem, node);
10014                 free(elem);
10015         }
10016
10017         /* Meter profiles. */
10018         for ( ; ; ) {
10019                 struct meter_profile *elem;
10020
10021                 elem = TAILQ_FIRST(&p->meter_profiles);
10022                 if (!elem)
10023                         break;
10024
10025                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
10026                 free(elem);
10027         }
10028 }
10029
10030 /*
10031  * Pipeline.
10032  */
10033 int
10034 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
10035 {
10036         struct rte_swx_pipeline *pipeline;
10037
10038         /* Check input parameters. */
10039         CHECK(p, EINVAL);
10040
10041         /* Memory allocation. */
10042         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
10043         CHECK(pipeline, ENOMEM);
10044
10045         /* Initialization. */
10046         TAILQ_INIT(&pipeline->struct_types);
10047         TAILQ_INIT(&pipeline->port_in_types);
10048         TAILQ_INIT(&pipeline->ports_in);
10049         TAILQ_INIT(&pipeline->port_out_types);
10050         TAILQ_INIT(&pipeline->ports_out);
10051         TAILQ_INIT(&pipeline->extern_types);
10052         TAILQ_INIT(&pipeline->extern_objs);
10053         TAILQ_INIT(&pipeline->extern_funcs);
10054         TAILQ_INIT(&pipeline->headers);
10055         TAILQ_INIT(&pipeline->actions);
10056         TAILQ_INIT(&pipeline->table_types);
10057         TAILQ_INIT(&pipeline->tables);
10058         TAILQ_INIT(&pipeline->selectors);
10059         TAILQ_INIT(&pipeline->learners);
10060         TAILQ_INIT(&pipeline->regarrays);
10061         TAILQ_INIT(&pipeline->meter_profiles);
10062         TAILQ_INIT(&pipeline->metarrays);
10063
10064         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
10065         pipeline->numa_node = numa_node;
10066
10067         *p = pipeline;
10068         return 0;
10069 }
10070
10071 void
10072 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
10073 {
10074         if (!p)
10075                 return;
10076
10077         free(p->instructions);
10078
10079         metarray_free(p);
10080         regarray_free(p);
10081         table_state_free(p);
10082         learner_free(p);
10083         selector_free(p);
10084         table_free(p);
10085         action_free(p);
10086         metadata_free(p);
10087         header_free(p);
10088         extern_func_free(p);
10089         extern_obj_free(p);
10090         port_out_free(p);
10091         port_in_free(p);
10092         struct_free(p);
10093
10094         free(p);
10095 }
10096
10097 int
10098 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
10099                                      const char **instructions,
10100                                      uint32_t n_instructions)
10101 {
10102         int err;
10103         uint32_t i;
10104
10105         err = instruction_config(p, NULL, instructions, n_instructions);
10106         if (err)
10107                 return err;
10108
10109         /* Thread instruction pointer reset. */
10110         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10111                 struct thread *t = &p->threads[i];
10112
10113                 thread_ip_reset(p, t);
10114         }
10115
10116         return 0;
10117 }
10118
10119 int
10120 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10121 {
10122         int status;
10123
10124         CHECK(p, EINVAL);
10125         CHECK(p->build_done == 0, EEXIST);
10126
10127         status = port_in_build(p);
10128         if (status)
10129                 goto error;
10130
10131         status = port_out_build(p);
10132         if (status)
10133                 goto error;
10134
10135         status = struct_build(p);
10136         if (status)
10137                 goto error;
10138
10139         status = extern_obj_build(p);
10140         if (status)
10141                 goto error;
10142
10143         status = extern_func_build(p);
10144         if (status)
10145                 goto error;
10146
10147         status = header_build(p);
10148         if (status)
10149                 goto error;
10150
10151         status = metadata_build(p);
10152         if (status)
10153                 goto error;
10154
10155         status = action_build(p);
10156         if (status)
10157                 goto error;
10158
10159         status = table_build(p);
10160         if (status)
10161                 goto error;
10162
10163         status = selector_build(p);
10164         if (status)
10165                 goto error;
10166
10167         status = learner_build(p);
10168         if (status)
10169                 goto error;
10170
10171         status = table_state_build(p);
10172         if (status)
10173                 goto error;
10174
10175         status = regarray_build(p);
10176         if (status)
10177                 goto error;
10178
10179         status = metarray_build(p);
10180         if (status)
10181                 goto error;
10182
10183         p->build_done = 1;
10184         return 0;
10185
10186 error:
10187         metarray_build_free(p);
10188         regarray_build_free(p);
10189         table_state_build_free(p);
10190         learner_build_free(p);
10191         selector_build_free(p);
10192         table_build_free(p);
10193         action_build_free(p);
10194         metadata_build_free(p);
10195         header_build_free(p);
10196         extern_func_build_free(p);
10197         extern_obj_build_free(p);
10198         port_out_build_free(p);
10199         port_in_build_free(p);
10200         struct_build_free(p);
10201
10202         return status;
10203 }
10204
10205 void
10206 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10207 {
10208         uint32_t i;
10209
10210         for (i = 0; i < n_instructions; i++)
10211                 instr_exec(p);
10212 }
10213
10214 void
10215 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10216 {
10217         uint32_t i;
10218
10219         for (i = 0; i < p->n_ports_out; i++) {
10220                 struct port_out_runtime *port = &p->out[i];
10221
10222                 if (port->flush)
10223                         port->flush(port->obj);
10224         }
10225 }
10226
10227 /*
10228  * Control.
10229  */
10230 int
10231 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10232                               struct rte_swx_ctl_pipeline_info *pipeline)
10233 {
10234         struct action *action;
10235         struct table *table;
10236         uint32_t n_actions = 0, n_tables = 0;
10237
10238         if (!p || !pipeline)
10239                 return -EINVAL;
10240
10241         TAILQ_FOREACH(action, &p->actions, node)
10242                 n_actions++;
10243
10244         TAILQ_FOREACH(table, &p->tables, node)
10245                 n_tables++;
10246
10247         pipeline->n_ports_in = p->n_ports_in;
10248         pipeline->n_ports_out = p->n_ports_out;
10249         pipeline->n_actions = n_actions;
10250         pipeline->n_tables = n_tables;
10251         pipeline->n_selectors = p->n_selectors;
10252         pipeline->n_learners = p->n_learners;
10253         pipeline->n_regarrays = p->n_regarrays;
10254         pipeline->n_metarrays = p->n_metarrays;
10255
10256         return 0;
10257 }
10258
10259 int
10260 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10261 {
10262         if (!p || !numa_node)
10263                 return -EINVAL;
10264
10265         *numa_node = p->numa_node;
10266         return 0;
10267 }
10268
10269 int
10270 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10271                             uint32_t action_id,
10272                             struct rte_swx_ctl_action_info *action)
10273 {
10274         struct action *a = NULL;
10275
10276         if (!p || (action_id >= p->n_actions) || !action)
10277                 return -EINVAL;
10278
10279         a = action_find_by_id(p, action_id);
10280         if (!a)
10281                 return -EINVAL;
10282
10283         strcpy(action->name, a->name);
10284         action->n_args = a->st ? a->st->n_fields : 0;
10285         return 0;
10286 }
10287
10288 int
10289 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10290                                 uint32_t action_id,
10291                                 uint32_t action_arg_id,
10292                                 struct rte_swx_ctl_action_arg_info *action_arg)
10293 {
10294         struct action *a = NULL;
10295         struct field *arg = NULL;
10296
10297         if (!p || (action_id >= p->n_actions) || !action_arg)
10298                 return -EINVAL;
10299
10300         a = action_find_by_id(p, action_id);
10301         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10302                 return -EINVAL;
10303
10304         arg = &a->st->fields[action_arg_id];
10305         strcpy(action_arg->name, arg->name);
10306         action_arg->n_bits = arg->n_bits;
10307         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10308
10309         return 0;
10310 }
10311
10312 int
10313 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10314                            uint32_t table_id,
10315                            struct rte_swx_ctl_table_info *table)
10316 {
10317         struct table *t = NULL;
10318
10319         if (!p || !table)
10320                 return -EINVAL;
10321
10322         t = table_find_by_id(p, table_id);
10323         if (!t)
10324                 return -EINVAL;
10325
10326         strcpy(table->name, t->name);
10327         strcpy(table->args, t->args);
10328         table->n_match_fields = t->n_fields;
10329         table->n_actions = t->n_actions;
10330         table->default_action_is_const = t->default_action_is_const;
10331         table->size = t->size;
10332         return 0;
10333 }
10334
10335 int
10336 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10337         uint32_t table_id,
10338         uint32_t match_field_id,
10339         struct rte_swx_ctl_table_match_field_info *match_field)
10340 {
10341         struct table *t;
10342         struct match_field *f;
10343
10344         if (!p || (table_id >= p->n_tables) || !match_field)
10345                 return -EINVAL;
10346
10347         t = table_find_by_id(p, table_id);
10348         if (!t || (match_field_id >= t->n_fields))
10349                 return -EINVAL;
10350
10351         f = &t->fields[match_field_id];
10352         match_field->match_type = f->match_type;
10353         match_field->is_header = t->header ? 1 : 0;
10354         match_field->n_bits = f->field->n_bits;
10355         match_field->offset = f->field->offset;
10356
10357         return 0;
10358 }
10359
10360 int
10361 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10362         uint32_t table_id,
10363         uint32_t table_action_id,
10364         struct rte_swx_ctl_table_action_info *table_action)
10365 {
10366         struct table *t;
10367
10368         if (!p || (table_id >= p->n_tables) || !table_action)
10369                 return -EINVAL;
10370
10371         t = table_find_by_id(p, table_id);
10372         if (!t || (table_action_id >= t->n_actions))
10373                 return -EINVAL;
10374
10375         table_action->action_id = t->actions[table_action_id]->id;
10376
10377         return 0;
10378 }
10379
10380 int
10381 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10382                           uint32_t table_id,
10383                           struct rte_swx_table_ops *table_ops,
10384                           int *is_stub)
10385 {
10386         struct table *t;
10387
10388         if (!p || (table_id >= p->n_tables))
10389                 return -EINVAL;
10390
10391         t = table_find_by_id(p, table_id);
10392         if (!t)
10393                 return -EINVAL;
10394
10395         if (t->type) {
10396                 if (table_ops)
10397                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10398                 *is_stub = 0;
10399         } else {
10400                 *is_stub = 1;
10401         }
10402
10403         return 0;
10404 }
10405
10406 int
10407 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10408                               uint32_t selector_id,
10409                               struct rte_swx_ctl_selector_info *selector)
10410 {
10411         struct selector *s = NULL;
10412
10413         if (!p || !selector)
10414                 return -EINVAL;
10415
10416         s = selector_find_by_id(p, selector_id);
10417         if (!s)
10418                 return -EINVAL;
10419
10420         strcpy(selector->name, s->name);
10421
10422         selector->n_selector_fields = s->n_selector_fields;
10423         selector->n_groups_max = s->n_groups_max;
10424         selector->n_members_per_group_max = s->n_members_per_group_max;
10425
10426         return 0;
10427 }
10428
10429 int
10430 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10431          uint32_t selector_id,
10432          struct rte_swx_ctl_table_match_field_info *field)
10433 {
10434         struct selector *s;
10435
10436         if (!p || (selector_id >= p->n_selectors) || !field)
10437                 return -EINVAL;
10438
10439         s = selector_find_by_id(p, selector_id);
10440         if (!s)
10441                 return -EINVAL;
10442
10443         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10444         field->is_header = 0;
10445         field->n_bits = s->group_id_field->n_bits;
10446         field->offset = s->group_id_field->offset;
10447
10448         return 0;
10449 }
10450
10451 int
10452 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10453          uint32_t selector_id,
10454          uint32_t selector_field_id,
10455          struct rte_swx_ctl_table_match_field_info *field)
10456 {
10457         struct selector *s;
10458         struct field *f;
10459
10460         if (!p || (selector_id >= p->n_selectors) || !field)
10461                 return -EINVAL;
10462
10463         s = selector_find_by_id(p, selector_id);
10464         if (!s || (selector_field_id >= s->n_selector_fields))
10465                 return -EINVAL;
10466
10467         f = s->selector_fields[selector_field_id];
10468         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10469         field->is_header = s->selector_header ? 1 : 0;
10470         field->n_bits = f->n_bits;
10471         field->offset = f->offset;
10472
10473         return 0;
10474 }
10475
10476 int
10477 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10478          uint32_t selector_id,
10479          struct rte_swx_ctl_table_match_field_info *field)
10480 {
10481         struct selector *s;
10482
10483         if (!p || (selector_id >= p->n_selectors) || !field)
10484                 return -EINVAL;
10485
10486         s = selector_find_by_id(p, selector_id);
10487         if (!s)
10488                 return -EINVAL;
10489
10490         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10491         field->is_header = 0;
10492         field->n_bits = s->member_id_field->n_bits;
10493         field->offset = s->member_id_field->offset;
10494
10495         return 0;
10496 }
10497
10498 int
10499 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10500                              uint32_t learner_id,
10501                              struct rte_swx_ctl_learner_info *learner)
10502 {
10503         struct learner *l = NULL;
10504
10505         if (!p || !learner)
10506                 return -EINVAL;
10507
10508         l = learner_find_by_id(p, learner_id);
10509         if (!l)
10510                 return -EINVAL;
10511
10512         strcpy(learner->name, l->name);
10513
10514         learner->n_match_fields = l->n_fields;
10515         learner->n_actions = l->n_actions;
10516         learner->default_action_is_const = l->default_action_is_const;
10517         learner->size = l->size;
10518
10519         return 0;
10520 }
10521
10522 int
10523 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10524                                          uint32_t learner_id,
10525                                          uint32_t match_field_id,
10526                                          struct rte_swx_ctl_table_match_field_info *match_field)
10527 {
10528         struct learner *l;
10529         struct field *f;
10530
10531         if (!p || (learner_id >= p->n_learners) || !match_field)
10532                 return -EINVAL;
10533
10534         l = learner_find_by_id(p, learner_id);
10535         if (!l || (match_field_id >= l->n_fields))
10536                 return -EINVAL;
10537
10538         f = l->fields[match_field_id];
10539         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10540         match_field->is_header = l->header ? 1 : 0;
10541         match_field->n_bits = f->n_bits;
10542         match_field->offset = f->offset;
10543
10544         return 0;
10545 }
10546
10547 int
10548 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10549                                     uint32_t learner_id,
10550                                     uint32_t learner_action_id,
10551                                     struct rte_swx_ctl_table_action_info *learner_action)
10552 {
10553         struct learner *l;
10554
10555         if (!p || (learner_id >= p->n_learners) || !learner_action)
10556                 return -EINVAL;
10557
10558         l = learner_find_by_id(p, learner_id);
10559         if (!l || (learner_action_id >= l->n_actions))
10560                 return -EINVAL;
10561
10562         learner_action->action_id = l->actions[learner_action_id]->id;
10563
10564         return 0;
10565 }
10566
10567 int
10568 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10569                                  struct rte_swx_table_state **table_state)
10570 {
10571         if (!p || !table_state || !p->build_done)
10572                 return -EINVAL;
10573
10574         *table_state = p->table_state;
10575         return 0;
10576 }
10577
10578 int
10579 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10580                                  struct rte_swx_table_state *table_state)
10581 {
10582         if (!p || !table_state || !p->build_done)
10583                 return -EINVAL;
10584
10585         p->table_state = table_state;
10586         return 0;
10587 }
10588
10589 int
10590 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10591                                         uint32_t port_id,
10592                                         struct rte_swx_port_in_stats *stats)
10593 {
10594         struct port_in *port;
10595
10596         if (!p || !stats)
10597                 return -EINVAL;
10598
10599         port = port_in_find(p, port_id);
10600         if (!port)
10601                 return -EINVAL;
10602
10603         port->type->ops.stats_read(port->obj, stats);
10604         return 0;
10605 }
10606
10607 int
10608 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10609                                          uint32_t port_id,
10610                                          struct rte_swx_port_out_stats *stats)
10611 {
10612         struct port_out *port;
10613
10614         if (!p || !stats)
10615                 return -EINVAL;
10616
10617         port = port_out_find(p, port_id);
10618         if (!port)
10619                 return -EINVAL;
10620
10621         port->type->ops.stats_read(port->obj, stats);
10622         return 0;
10623 }
10624
10625 int
10626 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10627                                       const char *table_name,
10628                                       struct rte_swx_table_stats *stats)
10629 {
10630         struct table *table;
10631         struct table_statistics *table_stats;
10632
10633         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10634                 return -EINVAL;
10635
10636         table = table_find(p, table_name);
10637         if (!table)
10638                 return -EINVAL;
10639
10640         table_stats = &p->table_stats[table->id];
10641
10642         memcpy(stats->n_pkts_action,
10643                table_stats->n_pkts_action,
10644                p->n_actions * sizeof(uint64_t));
10645
10646         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10647         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10648
10649         return 0;
10650 }
10651
10652 int
10653 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10654         const char *selector_name,
10655         struct rte_swx_pipeline_selector_stats *stats)
10656 {
10657         struct selector *s;
10658
10659         if (!p || !selector_name || !selector_name[0] || !stats)
10660                 return -EINVAL;
10661
10662         s = selector_find(p, selector_name);
10663         if (!s)
10664                 return -EINVAL;
10665
10666         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10667
10668         return 0;
10669 }
10670
10671 int
10672 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10673                                         const char *learner_name,
10674                                         struct rte_swx_learner_stats *stats)
10675 {
10676         struct learner *l;
10677         struct learner_statistics *learner_stats;
10678
10679         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10680                 return -EINVAL;
10681
10682         l = learner_find(p, learner_name);
10683         if (!l)
10684                 return -EINVAL;
10685
10686         learner_stats = &p->learner_stats[l->id];
10687
10688         memcpy(stats->n_pkts_action,
10689                learner_stats->n_pkts_action,
10690                p->n_actions * sizeof(uint64_t));
10691
10692         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10693         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10694
10695         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10696         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10697
10698         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10699
10700         return 0;
10701 }
10702
10703 int
10704 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10705                               uint32_t regarray_id,
10706                               struct rte_swx_ctl_regarray_info *regarray)
10707 {
10708         struct regarray *r;
10709
10710         if (!p || !regarray)
10711                 return -EINVAL;
10712
10713         r = regarray_find_by_id(p, regarray_id);
10714         if (!r)
10715                 return -EINVAL;
10716
10717         strcpy(regarray->name, r->name);
10718         regarray->size = r->size;
10719         return 0;
10720 }
10721
10722 int
10723 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10724                                    const char *regarray_name,
10725                                    uint32_t regarray_index,
10726                                    uint64_t *value)
10727 {
10728         struct regarray *regarray;
10729         struct regarray_runtime *r;
10730
10731         if (!p || !regarray_name || !value)
10732                 return -EINVAL;
10733
10734         regarray = regarray_find(p, regarray_name);
10735         if (!regarray || (regarray_index >= regarray->size))
10736                 return -EINVAL;
10737
10738         r = &p->regarray_runtime[regarray->id];
10739         *value = r->regarray[regarray_index];
10740         return 0;
10741 }
10742
10743 int
10744 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10745                                    const char *regarray_name,
10746                                    uint32_t regarray_index,
10747                                    uint64_t value)
10748 {
10749         struct regarray *regarray;
10750         struct regarray_runtime *r;
10751
10752         if (!p || !regarray_name)
10753                 return -EINVAL;
10754
10755         regarray = regarray_find(p, regarray_name);
10756         if (!regarray || (regarray_index >= regarray->size))
10757                 return -EINVAL;
10758
10759         r = &p->regarray_runtime[regarray->id];
10760         r->regarray[regarray_index] = value;
10761         return 0;
10762 }
10763
10764 int
10765 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10766                               uint32_t metarray_id,
10767                               struct rte_swx_ctl_metarray_info *metarray)
10768 {
10769         struct metarray *m;
10770
10771         if (!p || !metarray)
10772                 return -EINVAL;
10773
10774         m = metarray_find_by_id(p, metarray_id);
10775         if (!m)
10776                 return -EINVAL;
10777
10778         strcpy(metarray->name, m->name);
10779         metarray->size = m->size;
10780         return 0;
10781 }
10782
10783 int
10784 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10785                               const char *name,
10786                               struct rte_meter_trtcm_params *params)
10787 {
10788         struct meter_profile *mp;
10789         int status;
10790
10791         CHECK(p, EINVAL);
10792         CHECK_NAME(name, EINVAL);
10793         CHECK(params, EINVAL);
10794         CHECK(!meter_profile_find(p, name), EEXIST);
10795
10796         /* Node allocation. */
10797         mp = calloc(1, sizeof(struct meter_profile));
10798         CHECK(mp, ENOMEM);
10799
10800         /* Node initialization. */
10801         strcpy(mp->name, name);
10802         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10803         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10804         if (status) {
10805                 free(mp);
10806                 CHECK(0, EINVAL);
10807         }
10808
10809         /* Node add to tailq. */
10810         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10811
10812         return 0;
10813 }
10814
10815 int
10816 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10817                                  const char *name)
10818 {
10819         struct meter_profile *mp;
10820
10821         CHECK(p, EINVAL);
10822         CHECK_NAME(name, EINVAL);
10823
10824         mp = meter_profile_find(p, name);
10825         CHECK(mp, EINVAL);
10826         CHECK(!mp->n_users, EBUSY);
10827
10828         /* Remove node from tailq. */
10829         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10830         free(mp);
10831
10832         return 0;
10833 }
10834
10835 int
10836 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10837                         const char *metarray_name,
10838                         uint32_t metarray_index)
10839 {
10840         struct meter_profile *mp_old;
10841         struct metarray *metarray;
10842         struct metarray_runtime *metarray_runtime;
10843         struct meter *m;
10844
10845         CHECK(p, EINVAL);
10846         CHECK_NAME(metarray_name, EINVAL);
10847
10848         metarray = metarray_find(p, metarray_name);
10849         CHECK(metarray, EINVAL);
10850         CHECK(metarray_index < metarray->size, EINVAL);
10851
10852         metarray_runtime = &p->metarray_runtime[metarray->id];
10853         m = &metarray_runtime->metarray[metarray_index];
10854         mp_old = m->profile;
10855
10856         meter_init(m);
10857
10858         mp_old->n_users--;
10859
10860         return 0;
10861 }
10862
10863 int
10864 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10865                       const char *metarray_name,
10866                       uint32_t metarray_index,
10867                       const char *profile_name)
10868 {
10869         struct meter_profile *mp, *mp_old;
10870         struct metarray *metarray;
10871         struct metarray_runtime *metarray_runtime;
10872         struct meter *m;
10873
10874         CHECK(p, EINVAL);
10875         CHECK_NAME(metarray_name, EINVAL);
10876
10877         metarray = metarray_find(p, metarray_name);
10878         CHECK(metarray, EINVAL);
10879         CHECK(metarray_index < metarray->size, EINVAL);
10880
10881         mp = meter_profile_find(p, profile_name);
10882         CHECK(mp, EINVAL);
10883
10884         metarray_runtime = &p->metarray_runtime[metarray->id];
10885         m = &metarray_runtime->metarray[metarray_index];
10886         mp_old = m->profile;
10887
10888         memset(m, 0, sizeof(struct meter));
10889         rte_meter_trtcm_config(&m->m, &mp->profile);
10890         m->profile = mp;
10891         m->color_mask = RTE_COLORS;
10892
10893         mp->n_users++;
10894         mp_old->n_users--;
10895
10896         return 0;
10897 }
10898
10899 int
10900 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10901                              const char *metarray_name,
10902                              uint32_t metarray_index,
10903                              struct rte_swx_ctl_meter_stats *stats)
10904 {
10905         struct metarray *metarray;
10906         struct metarray_runtime *metarray_runtime;
10907         struct meter *m;
10908
10909         CHECK(p, EINVAL);
10910         CHECK_NAME(metarray_name, EINVAL);
10911
10912         metarray = metarray_find(p, metarray_name);
10913         CHECK(metarray, EINVAL);
10914         CHECK(metarray_index < metarray->size, EINVAL);
10915
10916         CHECK(stats, EINVAL);
10917
10918         metarray_runtime = &p->metarray_runtime[metarray->id];
10919         m = &metarray_runtime->metarray[metarray_index];
10920
10921         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10922         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10923
10924         return 0;
10925 }