lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 emit_handler(struct thread *t)
1572 {
1573         struct header_out_runtime *h0 = &t->headers_out[0];
1574         struct header_out_runtime *h1 = &t->headers_out[1];
1575         uint32_t offset = 0, i;
1576
1577         /* No header change or header decapsulation. */
1578         if ((t->n_headers_out == 1) &&
1579             (h0->ptr + h0->n_bytes == t->ptr)) {
1580                 TRACE("Emit handler: no header change or header decap.\n");
1581
1582                 t->pkt.offset -= h0->n_bytes;
1583                 t->pkt.length += h0->n_bytes;
1584
1585                 return;
1586         }
1587
1588         /* Header encapsulation (optionally, with prior header decasulation). */
1589         if ((t->n_headers_out == 2) &&
1590             (h1->ptr + h1->n_bytes == t->ptr) &&
1591             (h0->ptr == h0->ptr0)) {
1592                 uint32_t offset;
1593
1594                 TRACE("Emit handler: header encapsulation.\n");
1595
1596                 offset = h0->n_bytes + h1->n_bytes;
1597                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1598                 t->pkt.offset -= offset;
1599                 t->pkt.length += offset;
1600
1601                 return;
1602         }
1603
1604         /* Header insertion. */
1605         /* TBD */
1606
1607         /* Header extraction. */
1608         /* TBD */
1609
1610         /* For any other case. */
1611         TRACE("Emit handler: complex case.\n");
1612
1613         for (i = 0; i < t->n_headers_out; i++) {
1614                 struct header_out_runtime *h = &t->headers_out[i];
1615
1616                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1617                 offset += h->n_bytes;
1618         }
1619
1620         if (offset) {
1621                 memcpy(t->ptr - offset, t->header_out_storage, offset);
1622                 t->pkt.offset -= offset;
1623                 t->pkt.length += offset;
1624         }
1625 }
1626
1627 static inline void
1628 instr_tx_exec(struct rte_swx_pipeline *p);
1629
1630 static inline void
1631 instr_tx_exec(struct rte_swx_pipeline *p)
1632 {
1633         struct thread *t = &p->threads[p->thread_id];
1634         struct instruction *ip = t->ip;
1635         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1636         struct port_out_runtime *port = &p->out[port_id];
1637         struct rte_swx_pkt *pkt = &t->pkt;
1638
1639         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1640               p->thread_id,
1641               (uint32_t)port_id);
1642
1643         /* Headers. */
1644         emit_handler(t);
1645
1646         /* Packet. */
1647         port->pkt_tx(port->obj, pkt);
1648
1649         /* Thread. */
1650         thread_ip_reset(p, t);
1651         instr_rx_exec(p);
1652 }
1653
1654 static inline void
1655 instr_tx_i_exec(struct rte_swx_pipeline *p)
1656 {
1657         struct thread *t = &p->threads[p->thread_id];
1658         struct instruction *ip = t->ip;
1659         uint64_t port_id = ip->io.io.val;
1660         struct port_out_runtime *port = &p->out[port_id];
1661         struct rte_swx_pkt *pkt = &t->pkt;
1662
1663         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1664               p->thread_id,
1665               (uint32_t)port_id);
1666
1667         /* Headers. */
1668         emit_handler(t);
1669
1670         /* Packet. */
1671         port->pkt_tx(port->obj, pkt);
1672
1673         /* Thread. */
1674         thread_ip_reset(p, t);
1675         instr_rx_exec(p);
1676 }
1677
1678 /*
1679  * extract.
1680  */
1681 static int
1682 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1683                             struct action *action,
1684                             char **tokens,
1685                             int n_tokens,
1686                             struct instruction *instr,
1687                             struct instruction_data *data __rte_unused)
1688 {
1689         struct header *h;
1690
1691         CHECK(!action, EINVAL);
1692         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1693
1694         h = header_parse(p, tokens[1]);
1695         CHECK(h, EINVAL);
1696
1697         if (n_tokens == 2) {
1698                 CHECK(!h->st->var_size, EINVAL);
1699
1700                 instr->type = INSTR_HDR_EXTRACT;
1701                 instr->io.hdr.header_id[0] = h->id;
1702                 instr->io.hdr.struct_id[0] = h->struct_id;
1703                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1704         } else {
1705                 struct field *mf;
1706
1707                 CHECK(h->st->var_size, EINVAL);
1708
1709                 mf = metadata_field_parse(p, tokens[2]);
1710                 CHECK(mf, EINVAL);
1711                 CHECK(!mf->var_size, EINVAL);
1712
1713                 instr->type = INSTR_HDR_EXTRACT_M;
1714                 instr->io.io.offset = mf->offset / 8;
1715                 instr->io.io.n_bits = mf->n_bits;
1716                 instr->io.hdr.header_id[0] = h->id;
1717                 instr->io.hdr.struct_id[0] = h->struct_id;
1718                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1719         }
1720
1721         return 0;
1722 }
1723
1724 static int
1725 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1726                               struct action *action,
1727                               char **tokens,
1728                               int n_tokens,
1729                               struct instruction *instr,
1730                               struct instruction_data *data __rte_unused)
1731 {
1732         struct header *h;
1733
1734         CHECK(!action, EINVAL);
1735         CHECK(n_tokens == 2, EINVAL);
1736
1737         h = header_parse(p, tokens[1]);
1738         CHECK(h, EINVAL);
1739         CHECK(!h->st->var_size, EINVAL);
1740
1741         instr->type = INSTR_HDR_LOOKAHEAD;
1742         instr->io.hdr.header_id[0] = h->id;
1743         instr->io.hdr.struct_id[0] = h->struct_id;
1744         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1745
1746         return 0;
1747 }
1748
1749 static inline void
1750 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
1751
1752 static inline void
1753 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
1754 {
1755         struct thread *t = &p->threads[p->thread_id];
1756         struct instruction *ip = t->ip;
1757         uint64_t valid_headers = t->valid_headers;
1758         uint8_t *ptr = t->ptr;
1759         uint32_t offset = t->pkt.offset;
1760         uint32_t length = t->pkt.length;
1761         uint32_t i;
1762
1763         for (i = 0; i < n_extract; i++) {
1764                 uint32_t header_id = ip->io.hdr.header_id[i];
1765                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1766                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1767
1768                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1769                       p->thread_id,
1770                       header_id,
1771                       n_bytes);
1772
1773                 /* Headers. */
1774                 t->structs[struct_id] = ptr;
1775                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1776
1777                 /* Packet. */
1778                 offset += n_bytes;
1779                 length -= n_bytes;
1780                 ptr += n_bytes;
1781         }
1782
1783         /* Headers. */
1784         t->valid_headers = valid_headers;
1785
1786         /* Packet. */
1787         t->pkt.offset = offset;
1788         t->pkt.length = length;
1789         t->ptr = ptr;
1790 }
1791
1792 static inline void
1793 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1794 {
1795         __instr_hdr_extract_exec(p, 1);
1796
1797         /* Thread. */
1798         thread_ip_inc(p);
1799 }
1800
1801 static inline void
1802 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1803 {
1804         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
1805               p->thread_id);
1806
1807         __instr_hdr_extract_exec(p, 2);
1808
1809         /* Thread. */
1810         thread_ip_inc(p);
1811 }
1812
1813 static inline void
1814 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1815 {
1816         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
1817               p->thread_id);
1818
1819         __instr_hdr_extract_exec(p, 3);
1820
1821         /* Thread. */
1822         thread_ip_inc(p);
1823 }
1824
1825 static inline void
1826 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1827 {
1828         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
1829               p->thread_id);
1830
1831         __instr_hdr_extract_exec(p, 4);
1832
1833         /* Thread. */
1834         thread_ip_inc(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1839 {
1840         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
1841               p->thread_id);
1842
1843         __instr_hdr_extract_exec(p, 5);
1844
1845         /* Thread. */
1846         thread_ip_inc(p);
1847 }
1848
1849 static inline void
1850 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1851 {
1852         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
1853               p->thread_id);
1854
1855         __instr_hdr_extract_exec(p, 6);
1856
1857         /* Thread. */
1858         thread_ip_inc(p);
1859 }
1860
1861 static inline void
1862 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1863 {
1864         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
1865               p->thread_id);
1866
1867         __instr_hdr_extract_exec(p, 7);
1868
1869         /* Thread. */
1870         thread_ip_inc(p);
1871 }
1872
1873 static inline void
1874 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1875 {
1876         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
1877               p->thread_id);
1878
1879         __instr_hdr_extract_exec(p, 8);
1880
1881         /* Thread. */
1882         thread_ip_inc(p);
1883 }
1884
1885 static inline void
1886 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1887 {
1888         struct thread *t = &p->threads[p->thread_id];
1889         struct instruction *ip = t->ip;
1890
1891         uint64_t valid_headers = t->valid_headers;
1892         uint8_t *ptr = t->ptr;
1893         uint32_t offset = t->pkt.offset;
1894         uint32_t length = t->pkt.length;
1895
1896         uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1897         uint32_t header_id = ip->io.hdr.header_id[0];
1898         uint32_t struct_id = ip->io.hdr.struct_id[0];
1899         uint32_t n_bytes = ip->io.hdr.n_bytes[0];
1900
1901         struct header_runtime *h = &t->headers[header_id];
1902
1903         TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
1904               p->thread_id,
1905               header_id,
1906               n_bytes,
1907               n_bytes_last);
1908
1909         n_bytes += n_bytes_last;
1910
1911         /* Headers. */
1912         t->structs[struct_id] = ptr;
1913         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1914         h->n_bytes = n_bytes;
1915
1916         /* Packet. */
1917         t->pkt.offset = offset + n_bytes;
1918         t->pkt.length = length - n_bytes;
1919         t->ptr = ptr + n_bytes;
1920
1921         /* Thread. */
1922         thread_ip_inc(p);
1923 }
1924
1925 static inline void
1926 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1927 {
1928         struct thread *t = &p->threads[p->thread_id];
1929         struct instruction *ip = t->ip;
1930
1931         uint64_t valid_headers = t->valid_headers;
1932         uint8_t *ptr = t->ptr;
1933
1934         uint32_t header_id = ip->io.hdr.header_id[0];
1935         uint32_t struct_id = ip->io.hdr.struct_id[0];
1936
1937         TRACE("[Thread %2u]: lookahead header %u\n",
1938               p->thread_id,
1939               header_id);
1940
1941         /* Headers. */
1942         t->structs[struct_id] = ptr;
1943         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1944
1945         /* Thread. */
1946         thread_ip_inc(p);
1947 }
1948
1949 /*
1950  * emit.
1951  */
1952 static int
1953 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1954                          struct action *action __rte_unused,
1955                          char **tokens,
1956                          int n_tokens,
1957                          struct instruction *instr,
1958                          struct instruction_data *data __rte_unused)
1959 {
1960         struct header *h;
1961
1962         CHECK(n_tokens == 2, EINVAL);
1963
1964         h = header_parse(p, tokens[1]);
1965         CHECK(h, EINVAL);
1966
1967         instr->type = INSTR_HDR_EMIT;
1968         instr->io.hdr.header_id[0] = h->id;
1969         instr->io.hdr.struct_id[0] = h->struct_id;
1970         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1971         return 0;
1972 }
1973
1974 static inline void
1975 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
1976
1977 static inline void
1978 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
1979 {
1980         struct thread *t = &p->threads[p->thread_id];
1981         struct instruction *ip = t->ip;
1982         uint64_t valid_headers = t->valid_headers;
1983         uint32_t n_headers_out = t->n_headers_out;
1984         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1985         uint8_t *ho_ptr = NULL;
1986         uint32_t ho_nbytes = 0, first = 1, i;
1987
1988         for (i = 0; i < n_emit; i++) {
1989                 uint32_t header_id = ip->io.hdr.header_id[i];
1990                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1991
1992                 struct header_runtime *hi = &t->headers[header_id];
1993                 uint8_t *hi_ptr0 = hi->ptr0;
1994                 uint32_t n_bytes = hi->n_bytes;
1995
1996                 uint8_t *hi_ptr = t->structs[struct_id];
1997
1998                 if (!MASK64_BIT_GET(valid_headers, header_id))
1999                         continue;
2000
2001                 TRACE("[Thread %2u]: emit header %u\n",
2002                       p->thread_id,
2003                       header_id);
2004
2005                 /* Headers. */
2006                 if (first) {
2007                         first = 0;
2008
2009                         if (!t->n_headers_out) {
2010                                 ho = &t->headers_out[0];
2011
2012                                 ho->ptr0 = hi_ptr0;
2013                                 ho->ptr = hi_ptr;
2014
2015                                 ho_ptr = hi_ptr;
2016                                 ho_nbytes = n_bytes;
2017
2018                                 n_headers_out = 1;
2019
2020                                 continue;
2021                         } else {
2022                                 ho_ptr = ho->ptr;
2023                                 ho_nbytes = ho->n_bytes;
2024                         }
2025                 }
2026
2027                 if (ho_ptr + ho_nbytes == hi_ptr) {
2028                         ho_nbytes += n_bytes;
2029                 } else {
2030                         ho->n_bytes = ho_nbytes;
2031
2032                         ho++;
2033                         ho->ptr0 = hi_ptr0;
2034                         ho->ptr = hi_ptr;
2035
2036                         ho_ptr = hi_ptr;
2037                         ho_nbytes = n_bytes;
2038
2039                         n_headers_out++;
2040                 }
2041         }
2042
2043         ho->n_bytes = ho_nbytes;
2044         t->n_headers_out = n_headers_out;
2045 }
2046
2047 static inline void
2048 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2049 {
2050         __instr_hdr_emit_exec(p, 1);
2051
2052         /* Thread. */
2053         thread_ip_inc(p);
2054 }
2055
2056 static inline void
2057 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2058 {
2059         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2060               p->thread_id);
2061
2062         __instr_hdr_emit_exec(p, 1);
2063         instr_tx_exec(p);
2064 }
2065
2066 static inline void
2067 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2068 {
2069         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2070               p->thread_id);
2071
2072         __instr_hdr_emit_exec(p, 2);
2073         instr_tx_exec(p);
2074 }
2075
2076 static inline void
2077 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2078 {
2079         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2080               p->thread_id);
2081
2082         __instr_hdr_emit_exec(p, 3);
2083         instr_tx_exec(p);
2084 }
2085
2086 static inline void
2087 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2088 {
2089         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2090               p->thread_id);
2091
2092         __instr_hdr_emit_exec(p, 4);
2093         instr_tx_exec(p);
2094 }
2095
2096 static inline void
2097 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2098 {
2099         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2100               p->thread_id);
2101
2102         __instr_hdr_emit_exec(p, 5);
2103         instr_tx_exec(p);
2104 }
2105
2106 static inline void
2107 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2108 {
2109         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2110               p->thread_id);
2111
2112         __instr_hdr_emit_exec(p, 6);
2113         instr_tx_exec(p);
2114 }
2115
2116 static inline void
2117 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2118 {
2119         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2120               p->thread_id);
2121
2122         __instr_hdr_emit_exec(p, 7);
2123         instr_tx_exec(p);
2124 }
2125
2126 static inline void
2127 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2128 {
2129         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2130               p->thread_id);
2131
2132         __instr_hdr_emit_exec(p, 8);
2133         instr_tx_exec(p);
2134 }
2135
2136 /*
2137  * validate.
2138  */
2139 static int
2140 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2141                              struct action *action __rte_unused,
2142                              char **tokens,
2143                              int n_tokens,
2144                              struct instruction *instr,
2145                              struct instruction_data *data __rte_unused)
2146 {
2147         struct header *h;
2148
2149         CHECK(n_tokens == 2, EINVAL);
2150
2151         h = header_parse(p, tokens[1]);
2152         CHECK(h, EINVAL);
2153
2154         instr->type = INSTR_HDR_VALIDATE;
2155         instr->valid.header_id = h->id;
2156         return 0;
2157 }
2158
2159 static inline void
2160 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2161 {
2162         struct thread *t = &p->threads[p->thread_id];
2163         struct instruction *ip = t->ip;
2164         uint32_t header_id = ip->valid.header_id;
2165
2166         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2167
2168         /* Headers. */
2169         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2170
2171         /* Thread. */
2172         thread_ip_inc(p);
2173 }
2174
2175 /*
2176  * invalidate.
2177  */
2178 static int
2179 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2180                                struct action *action __rte_unused,
2181                                char **tokens,
2182                                int n_tokens,
2183                                struct instruction *instr,
2184                                struct instruction_data *data __rte_unused)
2185 {
2186         struct header *h;
2187
2188         CHECK(n_tokens == 2, EINVAL);
2189
2190         h = header_parse(p, tokens[1]);
2191         CHECK(h, EINVAL);
2192
2193         instr->type = INSTR_HDR_INVALIDATE;
2194         instr->valid.header_id = h->id;
2195         return 0;
2196 }
2197
2198 static inline void
2199 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2200 {
2201         struct thread *t = &p->threads[p->thread_id];
2202         struct instruction *ip = t->ip;
2203         uint32_t header_id = ip->valid.header_id;
2204
2205         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2206
2207         /* Headers. */
2208         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2209
2210         /* Thread. */
2211         thread_ip_inc(p);
2212 }
2213
2214 /*
2215  * table.
2216  */
2217 static struct table *
2218 table_find(struct rte_swx_pipeline *p, const char *name);
2219
2220 static struct selector *
2221 selector_find(struct rte_swx_pipeline *p, const char *name);
2222
2223 static struct learner *
2224 learner_find(struct rte_swx_pipeline *p, const char *name);
2225
2226 static int
2227 instr_table_translate(struct rte_swx_pipeline *p,
2228                       struct action *action,
2229                       char **tokens,
2230                       int n_tokens,
2231                       struct instruction *instr,
2232                       struct instruction_data *data __rte_unused)
2233 {
2234         struct table *t;
2235         struct selector *s;
2236         struct learner *l;
2237
2238         CHECK(!action, EINVAL);
2239         CHECK(n_tokens == 2, EINVAL);
2240
2241         t = table_find(p, tokens[1]);
2242         if (t) {
2243                 instr->type = INSTR_TABLE;
2244                 instr->table.table_id = t->id;
2245                 return 0;
2246         }
2247
2248         s = selector_find(p, tokens[1]);
2249         if (s) {
2250                 instr->type = INSTR_SELECTOR;
2251                 instr->table.table_id = s->id;
2252                 return 0;
2253         }
2254
2255         l = learner_find(p, tokens[1]);
2256         if (l) {
2257                 instr->type = INSTR_LEARNER;
2258                 instr->table.table_id = l->id;
2259                 return 0;
2260         }
2261
2262         CHECK(0, EINVAL);
2263 }
2264
2265 static inline void
2266 instr_table_exec(struct rte_swx_pipeline *p)
2267 {
2268         struct thread *t = &p->threads[p->thread_id];
2269         struct instruction *ip = t->ip;
2270         uint32_t table_id = ip->table.table_id;
2271         struct rte_swx_table_state *ts = &t->table_state[table_id];
2272         struct table_runtime *table = &t->tables[table_id];
2273         struct table_statistics *stats = &p->table_stats[table_id];
2274         uint64_t action_id, n_pkts_hit, n_pkts_action;
2275         uint8_t *action_data;
2276         int done, hit;
2277
2278         /* Table. */
2279         done = table->func(ts->obj,
2280                            table->mailbox,
2281                            table->key,
2282                            &action_id,
2283                            &action_data,
2284                            &hit);
2285         if (!done) {
2286                 /* Thread. */
2287                 TRACE("[Thread %2u] table %u (not finalized)\n",
2288                       p->thread_id,
2289                       table_id);
2290
2291                 thread_yield(p);
2292                 return;
2293         }
2294
2295         action_id = hit ? action_id : ts->default_action_id;
2296         action_data = hit ? action_data : ts->default_action_data;
2297         n_pkts_hit = stats->n_pkts_hit[hit];
2298         n_pkts_action = stats->n_pkts_action[action_id];
2299
2300         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2301               p->thread_id,
2302               table_id,
2303               hit ? "hit" : "miss",
2304               (uint32_t)action_id);
2305
2306         t->action_id = action_id;
2307         t->structs[0] = action_data;
2308         t->hit = hit;
2309         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2310         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2311
2312         /* Thread. */
2313         thread_ip_action_call(p, t, action_id);
2314 }
2315
2316 static inline void
2317 instr_selector_exec(struct rte_swx_pipeline *p)
2318 {
2319         struct thread *t = &p->threads[p->thread_id];
2320         struct instruction *ip = t->ip;
2321         uint32_t selector_id = ip->table.table_id;
2322         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2323         struct selector_runtime *selector = &t->selectors[selector_id];
2324         struct selector_statistics *stats = &p->selector_stats[selector_id];
2325         uint64_t n_pkts = stats->n_pkts;
2326         int done;
2327
2328         /* Table. */
2329         done = rte_swx_table_selector_select(ts->obj,
2330                            selector->mailbox,
2331                            selector->group_id_buffer,
2332                            selector->selector_buffer,
2333                            selector->member_id_buffer);
2334         if (!done) {
2335                 /* Thread. */
2336                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2337                       p->thread_id,
2338                       selector_id);
2339
2340                 thread_yield(p);
2341                 return;
2342         }
2343
2344
2345         TRACE("[Thread %2u] selector %u\n",
2346               p->thread_id,
2347               selector_id);
2348
2349         stats->n_pkts = n_pkts + 1;
2350
2351         /* Thread. */
2352         thread_ip_inc(p);
2353 }
2354
2355 static inline void
2356 instr_learner_exec(struct rte_swx_pipeline *p)
2357 {
2358         struct thread *t = &p->threads[p->thread_id];
2359         struct instruction *ip = t->ip;
2360         uint32_t learner_id = ip->table.table_id;
2361         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2362                 p->n_selectors + learner_id];
2363         struct learner_runtime *l = &t->learners[learner_id];
2364         struct learner_statistics *stats = &p->learner_stats[learner_id];
2365         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2366         uint8_t *action_data;
2367         int done, hit;
2368
2369         /* Table. */
2370         time = rte_get_tsc_cycles();
2371
2372         done = rte_swx_table_learner_lookup(ts->obj,
2373                                             l->mailbox,
2374                                             time,
2375                                             l->key,
2376                                             &action_id,
2377                                             &action_data,
2378                                             &hit);
2379         if (!done) {
2380                 /* Thread. */
2381                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2382                       p->thread_id,
2383                       learner_id);
2384
2385                 thread_yield(p);
2386                 return;
2387         }
2388
2389         action_id = hit ? action_id : ts->default_action_id;
2390         action_data = hit ? action_data : ts->default_action_data;
2391         n_pkts_hit = stats->n_pkts_hit[hit];
2392         n_pkts_action = stats->n_pkts_action[action_id];
2393
2394         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2395               p->thread_id,
2396               learner_id,
2397               hit ? "hit" : "miss",
2398               (uint32_t)action_id);
2399
2400         t->action_id = action_id;
2401         t->structs[0] = action_data;
2402         t->hit = hit;
2403         t->learner_id = learner_id;
2404         t->time = time;
2405         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2406         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2407
2408         /* Thread. */
2409         thread_ip_action_call(p, t, action_id);
2410 }
2411
2412 /*
2413  * learn.
2414  */
2415 static struct action *
2416 action_find(struct rte_swx_pipeline *p, const char *name);
2417
2418 static int
2419 action_has_nbo_args(struct action *a);
2420
2421 static int
2422 instr_learn_translate(struct rte_swx_pipeline *p,
2423                       struct action *action,
2424                       char **tokens,
2425                       int n_tokens,
2426                       struct instruction *instr,
2427                       struct instruction_data *data __rte_unused)
2428 {
2429         struct action *a;
2430
2431         CHECK(action, EINVAL);
2432         CHECK(n_tokens == 2, EINVAL);
2433
2434         a = action_find(p, tokens[1]);
2435         CHECK(a, EINVAL);
2436         CHECK(!action_has_nbo_args(a), EINVAL);
2437
2438         instr->type = INSTR_LEARNER_LEARN;
2439         instr->learn.action_id = a->id;
2440
2441         return 0;
2442 }
2443
2444 static inline void
2445 instr_learn_exec(struct rte_swx_pipeline *p)
2446 {
2447         struct thread *t = &p->threads[p->thread_id];
2448         struct instruction *ip = t->ip;
2449         uint64_t action_id = ip->learn.action_id;
2450         uint32_t learner_id = t->learner_id;
2451         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2452                 p->n_selectors + learner_id];
2453         struct learner_runtime *l = &t->learners[learner_id];
2454         struct learner_statistics *stats = &p->learner_stats[learner_id];
2455         uint32_t status;
2456
2457         /* Table. */
2458         status = rte_swx_table_learner_add(ts->obj,
2459                                            l->mailbox,
2460                                            t->time,
2461                                            action_id,
2462                                            l->action_data[action_id]);
2463
2464         TRACE("[Thread %2u] learner %u learn %s\n",
2465               p->thread_id,
2466               learner_id,
2467               status ? "ok" : "error");
2468
2469         stats->n_pkts_learn[status] += 1;
2470
2471         /* Thread. */
2472         thread_ip_inc(p);
2473 }
2474
2475 /*
2476  * forget.
2477  */
2478 static int
2479 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2480                        struct action *action,
2481                        char **tokens __rte_unused,
2482                        int n_tokens,
2483                        struct instruction *instr,
2484                        struct instruction_data *data __rte_unused)
2485 {
2486         CHECK(action, EINVAL);
2487         CHECK(n_tokens == 1, EINVAL);
2488
2489         instr->type = INSTR_LEARNER_FORGET;
2490
2491         return 0;
2492 }
2493
2494 static inline void
2495 instr_forget_exec(struct rte_swx_pipeline *p)
2496 {
2497         struct thread *t = &p->threads[p->thread_id];
2498         uint32_t learner_id = t->learner_id;
2499         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2500                 p->n_selectors + learner_id];
2501         struct learner_runtime *l = &t->learners[learner_id];
2502         struct learner_statistics *stats = &p->learner_stats[learner_id];
2503
2504         /* Table. */
2505         rte_swx_table_learner_delete(ts->obj, l->mailbox);
2506
2507         TRACE("[Thread %2u] learner %u forget\n",
2508               p->thread_id,
2509               learner_id);
2510
2511         stats->n_pkts_forget += 1;
2512
2513         /* Thread. */
2514         thread_ip_inc(p);
2515 }
2516
2517 /*
2518  * extern.
2519  */
2520 static int
2521 instr_extern_translate(struct rte_swx_pipeline *p,
2522                        struct action *action __rte_unused,
2523                        char **tokens,
2524                        int n_tokens,
2525                        struct instruction *instr,
2526                        struct instruction_data *data __rte_unused)
2527 {
2528         char *token = tokens[1];
2529
2530         CHECK(n_tokens == 2, EINVAL);
2531
2532         if (token[0] == 'e') {
2533                 struct extern_obj *obj;
2534                 struct extern_type_member_func *func;
2535
2536                 func = extern_obj_member_func_parse(p, token, &obj);
2537                 CHECK(func, EINVAL);
2538
2539                 instr->type = INSTR_EXTERN_OBJ;
2540                 instr->ext_obj.ext_obj_id = obj->id;
2541                 instr->ext_obj.func_id = func->id;
2542
2543                 return 0;
2544         }
2545
2546         if (token[0] == 'f') {
2547                 struct extern_func *func;
2548
2549                 func = extern_func_parse(p, token);
2550                 CHECK(func, EINVAL);
2551
2552                 instr->type = INSTR_EXTERN_FUNC;
2553                 instr->ext_func.ext_func_id = func->id;
2554
2555                 return 0;
2556         }
2557
2558         CHECK(0, EINVAL);
2559 }
2560
2561 static inline void
2562 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2563 {
2564         struct thread *t = &p->threads[p->thread_id];
2565         struct instruction *ip = t->ip;
2566         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2567         uint32_t func_id = ip->ext_obj.func_id;
2568         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2569         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2570
2571         TRACE("[Thread %2u] extern obj %u member func %u\n",
2572               p->thread_id,
2573               obj_id,
2574               func_id);
2575
2576         /* Extern object member function execute. */
2577         uint32_t done = func(obj->obj, obj->mailbox);
2578
2579         /* Thread. */
2580         thread_ip_inc_cond(t, done);
2581         thread_yield_cond(p, done ^ 1);
2582 }
2583
2584 static inline void
2585 instr_extern_func_exec(struct rte_swx_pipeline *p)
2586 {
2587         struct thread *t = &p->threads[p->thread_id];
2588         struct instruction *ip = t->ip;
2589         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2590         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2591         rte_swx_extern_func_t func = ext_func->func;
2592
2593         TRACE("[Thread %2u] extern func %u\n",
2594               p->thread_id,
2595               ext_func_id);
2596
2597         /* Extern function execute. */
2598         uint32_t done = func(ext_func->mailbox);
2599
2600         /* Thread. */
2601         thread_ip_inc_cond(t, done);
2602         thread_yield_cond(p, done ^ 1);
2603 }
2604
2605 /*
2606  * mov.
2607  */
2608 static int
2609 instr_mov_translate(struct rte_swx_pipeline *p,
2610                     struct action *action,
2611                     char **tokens,
2612                     int n_tokens,
2613                     struct instruction *instr,
2614                     struct instruction_data *data __rte_unused)
2615 {
2616         char *dst = tokens[1], *src = tokens[2];
2617         struct field *fdst, *fsrc;
2618         uint64_t src_val;
2619         uint32_t dst_struct_id = 0, src_struct_id = 0;
2620
2621         CHECK(n_tokens == 3, EINVAL);
2622
2623         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2624         CHECK(fdst, EINVAL);
2625         CHECK(!fdst->var_size, EINVAL);
2626
2627         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2628         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2629         if (fsrc) {
2630                 CHECK(!fsrc->var_size, EINVAL);
2631
2632                 instr->type = INSTR_MOV;
2633                 if (dst[0] != 'h' && src[0] == 'h')
2634                         instr->type = INSTR_MOV_MH;
2635                 if (dst[0] == 'h' && src[0] != 'h')
2636                         instr->type = INSTR_MOV_HM;
2637                 if (dst[0] == 'h' && src[0] == 'h')
2638                         instr->type = INSTR_MOV_HH;
2639
2640                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2641                 instr->mov.dst.n_bits = fdst->n_bits;
2642                 instr->mov.dst.offset = fdst->offset / 8;
2643                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2644                 instr->mov.src.n_bits = fsrc->n_bits;
2645                 instr->mov.src.offset = fsrc->offset / 8;
2646                 return 0;
2647         }
2648
2649         /* MOV_I. */
2650         src_val = strtoull(src, &src, 0);
2651         CHECK(!src[0], EINVAL);
2652
2653         if (dst[0] == 'h')
2654                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2655
2656         instr->type = INSTR_MOV_I;
2657         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2658         instr->mov.dst.n_bits = fdst->n_bits;
2659         instr->mov.dst.offset = fdst->offset / 8;
2660         instr->mov.src_val = src_val;
2661         return 0;
2662 }
2663
2664 static inline void
2665 instr_mov_exec(struct rte_swx_pipeline *p)
2666 {
2667         struct thread *t = &p->threads[p->thread_id];
2668         struct instruction *ip = t->ip;
2669
2670         TRACE("[Thread %2u] mov\n",
2671               p->thread_id);
2672
2673         MOV(t, ip);
2674
2675         /* Thread. */
2676         thread_ip_inc(p);
2677 }
2678
2679 static inline void
2680 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2681 {
2682         struct thread *t = &p->threads[p->thread_id];
2683         struct instruction *ip = t->ip;
2684
2685         TRACE("[Thread %2u] mov (mh)\n",
2686               p->thread_id);
2687
2688         MOV_MH(t, ip);
2689
2690         /* Thread. */
2691         thread_ip_inc(p);
2692 }
2693
2694 static inline void
2695 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2696 {
2697         struct thread *t = &p->threads[p->thread_id];
2698         struct instruction *ip = t->ip;
2699
2700         TRACE("[Thread %2u] mov (hm)\n",
2701               p->thread_id);
2702
2703         MOV_HM(t, ip);
2704
2705         /* Thread. */
2706         thread_ip_inc(p);
2707 }
2708
2709 static inline void
2710 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2711 {
2712         struct thread *t = &p->threads[p->thread_id];
2713         struct instruction *ip = t->ip;
2714
2715         TRACE("[Thread %2u] mov (hh)\n",
2716               p->thread_id);
2717
2718         MOV_HH(t, ip);
2719
2720         /* Thread. */
2721         thread_ip_inc(p);
2722 }
2723
2724 static inline void
2725 instr_mov_i_exec(struct rte_swx_pipeline *p)
2726 {
2727         struct thread *t = &p->threads[p->thread_id];
2728         struct instruction *ip = t->ip;
2729
2730         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2731               p->thread_id,
2732               ip->mov.src_val);
2733
2734         MOV_I(t, ip);
2735
2736         /* Thread. */
2737         thread_ip_inc(p);
2738 }
2739
2740 /*
2741  * dma.
2742  */
2743 static inline void
2744 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2745
2746 static inline void
2747 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2748 {
2749         struct thread *t = &p->threads[p->thread_id];
2750         struct instruction *ip = t->ip;
2751         uint8_t *action_data = t->structs[0];
2752         uint64_t valid_headers = t->valid_headers;
2753         uint32_t i;
2754
2755         for (i = 0; i < n_dma; i++) {
2756                 uint32_t header_id = ip->dma.dst.header_id[i];
2757                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2758                 uint32_t offset = ip->dma.src.offset[i];
2759                 uint32_t n_bytes = ip->dma.n_bytes[i];
2760
2761                 struct header_runtime *h = &t->headers[header_id];
2762                 uint8_t *h_ptr0 = h->ptr0;
2763                 uint8_t *h_ptr = t->structs[struct_id];
2764
2765                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2766                         h_ptr : h_ptr0;
2767                 void *src = &action_data[offset];
2768
2769                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2770
2771                 /* Headers. */
2772                 memcpy(dst, src, n_bytes);
2773                 t->structs[struct_id] = dst;
2774                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2775         }
2776
2777         t->valid_headers = valid_headers;
2778 }
2779
2780 static inline void
2781 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2782 {
2783         __instr_dma_ht_exec(p, 1);
2784
2785         /* Thread. */
2786         thread_ip_inc(p);
2787 }
2788
2789 static inline void
2790 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2791 {
2792         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2793               p->thread_id);
2794
2795         __instr_dma_ht_exec(p, 2);
2796
2797         /* Thread. */
2798         thread_ip_inc(p);
2799 }
2800
2801 static inline void
2802 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2803 {
2804         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2805               p->thread_id);
2806
2807         __instr_dma_ht_exec(p, 3);
2808
2809         /* Thread. */
2810         thread_ip_inc(p);
2811 }
2812
2813 static inline void
2814 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2815 {
2816         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2817               p->thread_id);
2818
2819         __instr_dma_ht_exec(p, 4);
2820
2821         /* Thread. */
2822         thread_ip_inc(p);
2823 }
2824
2825 static inline void
2826 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2827 {
2828         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2829               p->thread_id);
2830
2831         __instr_dma_ht_exec(p, 5);
2832
2833         /* Thread. */
2834         thread_ip_inc(p);
2835 }
2836
2837 static inline void
2838 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2839 {
2840         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2841               p->thread_id);
2842
2843         __instr_dma_ht_exec(p, 6);
2844
2845         /* Thread. */
2846         thread_ip_inc(p);
2847 }
2848
2849 static inline void
2850 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2851 {
2852         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2853               p->thread_id);
2854
2855         __instr_dma_ht_exec(p, 7);
2856
2857         /* Thread. */
2858         thread_ip_inc(p);
2859 }
2860
2861 static inline void
2862 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2863 {
2864         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2865               p->thread_id);
2866
2867         __instr_dma_ht_exec(p, 8);
2868
2869         /* Thread. */
2870         thread_ip_inc(p);
2871 }
2872
2873 /*
2874  * alu.
2875  */
2876 static int
2877 instr_alu_add_translate(struct rte_swx_pipeline *p,
2878                         struct action *action,
2879                         char **tokens,
2880                         int n_tokens,
2881                         struct instruction *instr,
2882                         struct instruction_data *data __rte_unused)
2883 {
2884         char *dst = tokens[1], *src = tokens[2];
2885         struct field *fdst, *fsrc;
2886         uint64_t src_val;
2887         uint32_t dst_struct_id = 0, src_struct_id = 0;
2888
2889         CHECK(n_tokens == 3, EINVAL);
2890
2891         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2892         CHECK(fdst, EINVAL);
2893         CHECK(!fdst->var_size, EINVAL);
2894
2895         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2896         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2897         if (fsrc) {
2898                 CHECK(!fsrc->var_size, EINVAL);
2899
2900                 instr->type = INSTR_ALU_ADD;
2901                 if (dst[0] == 'h' && src[0] != 'h')
2902                         instr->type = INSTR_ALU_ADD_HM;
2903                 if (dst[0] != 'h' && src[0] == 'h')
2904                         instr->type = INSTR_ALU_ADD_MH;
2905                 if (dst[0] == 'h' && src[0] == 'h')
2906                         instr->type = INSTR_ALU_ADD_HH;
2907
2908                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2909                 instr->alu.dst.n_bits = fdst->n_bits;
2910                 instr->alu.dst.offset = fdst->offset / 8;
2911                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2912                 instr->alu.src.n_bits = fsrc->n_bits;
2913                 instr->alu.src.offset = fsrc->offset / 8;
2914                 return 0;
2915         }
2916
2917         /* ADD_MI, ADD_HI. */
2918         src_val = strtoull(src, &src, 0);
2919         CHECK(!src[0], EINVAL);
2920
2921         instr->type = INSTR_ALU_ADD_MI;
2922         if (dst[0] == 'h')
2923                 instr->type = INSTR_ALU_ADD_HI;
2924
2925         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2926         instr->alu.dst.n_bits = fdst->n_bits;
2927         instr->alu.dst.offset = fdst->offset / 8;
2928         instr->alu.src_val = src_val;
2929         return 0;
2930 }
2931
2932 static int
2933 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2934                         struct action *action,
2935                         char **tokens,
2936                         int n_tokens,
2937                         struct instruction *instr,
2938                         struct instruction_data *data __rte_unused)
2939 {
2940         char *dst = tokens[1], *src = tokens[2];
2941         struct field *fdst, *fsrc;
2942         uint64_t src_val;
2943         uint32_t dst_struct_id = 0, src_struct_id = 0;
2944
2945         CHECK(n_tokens == 3, EINVAL);
2946
2947         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2948         CHECK(fdst, EINVAL);
2949         CHECK(!fdst->var_size, EINVAL);
2950
2951         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2952         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2953         if (fsrc) {
2954                 CHECK(!fsrc->var_size, EINVAL);
2955
2956                 instr->type = INSTR_ALU_SUB;
2957                 if (dst[0] == 'h' && src[0] != 'h')
2958                         instr->type = INSTR_ALU_SUB_HM;
2959                 if (dst[0] != 'h' && src[0] == 'h')
2960                         instr->type = INSTR_ALU_SUB_MH;
2961                 if (dst[0] == 'h' && src[0] == 'h')
2962                         instr->type = INSTR_ALU_SUB_HH;
2963
2964                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2965                 instr->alu.dst.n_bits = fdst->n_bits;
2966                 instr->alu.dst.offset = fdst->offset / 8;
2967                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2968                 instr->alu.src.n_bits = fsrc->n_bits;
2969                 instr->alu.src.offset = fsrc->offset / 8;
2970                 return 0;
2971         }
2972
2973         /* SUB_MI, SUB_HI. */
2974         src_val = strtoull(src, &src, 0);
2975         CHECK(!src[0], EINVAL);
2976
2977         instr->type = INSTR_ALU_SUB_MI;
2978         if (dst[0] == 'h')
2979                 instr->type = INSTR_ALU_SUB_HI;
2980
2981         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2982         instr->alu.dst.n_bits = fdst->n_bits;
2983         instr->alu.dst.offset = fdst->offset / 8;
2984         instr->alu.src_val = src_val;
2985         return 0;
2986 }
2987
2988 static int
2989 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2990                           struct action *action __rte_unused,
2991                           char **tokens,
2992                           int n_tokens,
2993                           struct instruction *instr,
2994                           struct instruction_data *data __rte_unused)
2995 {
2996         char *dst = tokens[1], *src = tokens[2];
2997         struct header *hdst, *hsrc;
2998         struct field *fdst, *fsrc;
2999
3000         CHECK(n_tokens == 3, EINVAL);
3001
3002         fdst = header_field_parse(p, dst, &hdst);
3003         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3004         CHECK(!fdst->var_size, EINVAL);
3005
3006         /* CKADD_FIELD. */
3007         fsrc = header_field_parse(p, src, &hsrc);
3008         if (fsrc) {
3009                 CHECK(!fsrc->var_size, EINVAL);
3010
3011                 instr->type = INSTR_ALU_CKADD_FIELD;
3012                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3013                 instr->alu.dst.n_bits = fdst->n_bits;
3014                 instr->alu.dst.offset = fdst->offset / 8;
3015                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3016                 instr->alu.src.n_bits = fsrc->n_bits;
3017                 instr->alu.src.offset = fsrc->offset / 8;
3018                 return 0;
3019         }
3020
3021         /* CKADD_STRUCT, CKADD_STRUCT20. */
3022         hsrc = header_parse(p, src);
3023         CHECK(hsrc, EINVAL);
3024         CHECK(!hsrc->st->var_size, EINVAL);
3025
3026         instr->type = INSTR_ALU_CKADD_STRUCT;
3027         if ((hsrc->st->n_bits / 8) == 20)
3028                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3029
3030         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3031         instr->alu.dst.n_bits = fdst->n_bits;
3032         instr->alu.dst.offset = fdst->offset / 8;
3033         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3034         instr->alu.src.n_bits = hsrc->st->n_bits;
3035         instr->alu.src.offset = 0; /* Unused. */
3036         return 0;
3037 }
3038
3039 static int
3040 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3041                           struct action *action __rte_unused,
3042                           char **tokens,
3043                           int n_tokens,
3044                           struct instruction *instr,
3045                           struct instruction_data *data __rte_unused)
3046 {
3047         char *dst = tokens[1], *src = tokens[2];
3048         struct header *hdst, *hsrc;
3049         struct field *fdst, *fsrc;
3050
3051         CHECK(n_tokens == 3, EINVAL);
3052
3053         fdst = header_field_parse(p, dst, &hdst);
3054         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3055         CHECK(!fdst->var_size, EINVAL);
3056
3057         fsrc = header_field_parse(p, src, &hsrc);
3058         CHECK(fsrc, EINVAL);
3059         CHECK(!fsrc->var_size, EINVAL);
3060
3061         instr->type = INSTR_ALU_CKSUB_FIELD;
3062         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3063         instr->alu.dst.n_bits = fdst->n_bits;
3064         instr->alu.dst.offset = fdst->offset / 8;
3065         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3066         instr->alu.src.n_bits = fsrc->n_bits;
3067         instr->alu.src.offset = fsrc->offset / 8;
3068         return 0;
3069 }
3070
3071 static int
3072 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3073                         struct action *action,
3074                         char **tokens,
3075                         int n_tokens,
3076                         struct instruction *instr,
3077                         struct instruction_data *data __rte_unused)
3078 {
3079         char *dst = tokens[1], *src = tokens[2];
3080         struct field *fdst, *fsrc;
3081         uint64_t src_val;
3082         uint32_t dst_struct_id = 0, src_struct_id = 0;
3083
3084         CHECK(n_tokens == 3, EINVAL);
3085
3086         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3087         CHECK(fdst, EINVAL);
3088         CHECK(!fdst->var_size, EINVAL);
3089
3090         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3091         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3092         if (fsrc) {
3093                 CHECK(!fsrc->var_size, EINVAL);
3094
3095                 instr->type = INSTR_ALU_SHL;
3096                 if (dst[0] == 'h' && src[0] != 'h')
3097                         instr->type = INSTR_ALU_SHL_HM;
3098                 if (dst[0] != 'h' && src[0] == 'h')
3099                         instr->type = INSTR_ALU_SHL_MH;
3100                 if (dst[0] == 'h' && src[0] == 'h')
3101                         instr->type = INSTR_ALU_SHL_HH;
3102
3103                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3104                 instr->alu.dst.n_bits = fdst->n_bits;
3105                 instr->alu.dst.offset = fdst->offset / 8;
3106                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3107                 instr->alu.src.n_bits = fsrc->n_bits;
3108                 instr->alu.src.offset = fsrc->offset / 8;
3109                 return 0;
3110         }
3111
3112         /* SHL_MI, SHL_HI. */
3113         src_val = strtoull(src, &src, 0);
3114         CHECK(!src[0], EINVAL);
3115
3116         instr->type = INSTR_ALU_SHL_MI;
3117         if (dst[0] == 'h')
3118                 instr->type = INSTR_ALU_SHL_HI;
3119
3120         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3121         instr->alu.dst.n_bits = fdst->n_bits;
3122         instr->alu.dst.offset = fdst->offset / 8;
3123         instr->alu.src_val = src_val;
3124         return 0;
3125 }
3126
3127 static int
3128 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3129                         struct action *action,
3130                         char **tokens,
3131                         int n_tokens,
3132                         struct instruction *instr,
3133                         struct instruction_data *data __rte_unused)
3134 {
3135         char *dst = tokens[1], *src = tokens[2];
3136         struct field *fdst, *fsrc;
3137         uint64_t src_val;
3138         uint32_t dst_struct_id = 0, src_struct_id = 0;
3139
3140         CHECK(n_tokens == 3, EINVAL);
3141
3142         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3143         CHECK(fdst, EINVAL);
3144         CHECK(!fdst->var_size, EINVAL);
3145
3146         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3147         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3148         if (fsrc) {
3149                 CHECK(!fsrc->var_size, EINVAL);
3150
3151                 instr->type = INSTR_ALU_SHR;
3152                 if (dst[0] == 'h' && src[0] != 'h')
3153                         instr->type = INSTR_ALU_SHR_HM;
3154                 if (dst[0] != 'h' && src[0] == 'h')
3155                         instr->type = INSTR_ALU_SHR_MH;
3156                 if (dst[0] == 'h' && src[0] == 'h')
3157                         instr->type = INSTR_ALU_SHR_HH;
3158
3159                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3160                 instr->alu.dst.n_bits = fdst->n_bits;
3161                 instr->alu.dst.offset = fdst->offset / 8;
3162                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3163                 instr->alu.src.n_bits = fsrc->n_bits;
3164                 instr->alu.src.offset = fsrc->offset / 8;
3165                 return 0;
3166         }
3167
3168         /* SHR_MI, SHR_HI. */
3169         src_val = strtoull(src, &src, 0);
3170         CHECK(!src[0], EINVAL);
3171
3172         instr->type = INSTR_ALU_SHR_MI;
3173         if (dst[0] == 'h')
3174                 instr->type = INSTR_ALU_SHR_HI;
3175
3176         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3177         instr->alu.dst.n_bits = fdst->n_bits;
3178         instr->alu.dst.offset = fdst->offset / 8;
3179         instr->alu.src_val = src_val;
3180         return 0;
3181 }
3182
3183 static int
3184 instr_alu_and_translate(struct rte_swx_pipeline *p,
3185                         struct action *action,
3186                         char **tokens,
3187                         int n_tokens,
3188                         struct instruction *instr,
3189                         struct instruction_data *data __rte_unused)
3190 {
3191         char *dst = tokens[1], *src = tokens[2];
3192         struct field *fdst, *fsrc;
3193         uint64_t src_val;
3194         uint32_t dst_struct_id = 0, src_struct_id = 0;
3195
3196         CHECK(n_tokens == 3, EINVAL);
3197
3198         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3199         CHECK(fdst, EINVAL);
3200         CHECK(!fdst->var_size, EINVAL);
3201
3202         /* AND, AND_MH, AND_HM, AND_HH. */
3203         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3204         if (fsrc) {
3205                 CHECK(!fsrc->var_size, EINVAL);
3206
3207                 instr->type = INSTR_ALU_AND;
3208                 if (dst[0] != 'h' && src[0] == 'h')
3209                         instr->type = INSTR_ALU_AND_MH;
3210                 if (dst[0] == 'h' && src[0] != 'h')
3211                         instr->type = INSTR_ALU_AND_HM;
3212                 if (dst[0] == 'h' && src[0] == 'h')
3213                         instr->type = INSTR_ALU_AND_HH;
3214
3215                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3216                 instr->alu.dst.n_bits = fdst->n_bits;
3217                 instr->alu.dst.offset = fdst->offset / 8;
3218                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3219                 instr->alu.src.n_bits = fsrc->n_bits;
3220                 instr->alu.src.offset = fsrc->offset / 8;
3221                 return 0;
3222         }
3223
3224         /* AND_I. */
3225         src_val = strtoull(src, &src, 0);
3226         CHECK(!src[0], EINVAL);
3227
3228         if (dst[0] == 'h')
3229                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3230
3231         instr->type = INSTR_ALU_AND_I;
3232         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3233         instr->alu.dst.n_bits = fdst->n_bits;
3234         instr->alu.dst.offset = fdst->offset / 8;
3235         instr->alu.src_val = src_val;
3236         return 0;
3237 }
3238
3239 static int
3240 instr_alu_or_translate(struct rte_swx_pipeline *p,
3241                        struct action *action,
3242                        char **tokens,
3243                        int n_tokens,
3244                        struct instruction *instr,
3245                        struct instruction_data *data __rte_unused)
3246 {
3247         char *dst = tokens[1], *src = tokens[2];
3248         struct field *fdst, *fsrc;
3249         uint64_t src_val;
3250         uint32_t dst_struct_id = 0, src_struct_id = 0;
3251
3252         CHECK(n_tokens == 3, EINVAL);
3253
3254         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3255         CHECK(fdst, EINVAL);
3256         CHECK(!fdst->var_size, EINVAL);
3257
3258         /* OR, OR_MH, OR_HM, OR_HH. */
3259         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3260         if (fsrc) {
3261                 CHECK(!fsrc->var_size, EINVAL);
3262
3263                 instr->type = INSTR_ALU_OR;
3264                 if (dst[0] != 'h' && src[0] == 'h')
3265                         instr->type = INSTR_ALU_OR_MH;
3266                 if (dst[0] == 'h' && src[0] != 'h')
3267                         instr->type = INSTR_ALU_OR_HM;
3268                 if (dst[0] == 'h' && src[0] == 'h')
3269                         instr->type = INSTR_ALU_OR_HH;
3270
3271                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3272                 instr->alu.dst.n_bits = fdst->n_bits;
3273                 instr->alu.dst.offset = fdst->offset / 8;
3274                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3275                 instr->alu.src.n_bits = fsrc->n_bits;
3276                 instr->alu.src.offset = fsrc->offset / 8;
3277                 return 0;
3278         }
3279
3280         /* OR_I. */
3281         src_val = strtoull(src, &src, 0);
3282         CHECK(!src[0], EINVAL);
3283
3284         if (dst[0] == 'h')
3285                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3286
3287         instr->type = INSTR_ALU_OR_I;
3288         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3289         instr->alu.dst.n_bits = fdst->n_bits;
3290         instr->alu.dst.offset = fdst->offset / 8;
3291         instr->alu.src_val = src_val;
3292         return 0;
3293 }
3294
3295 static int
3296 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3297                         struct action *action,
3298                         char **tokens,
3299                         int n_tokens,
3300                         struct instruction *instr,
3301                         struct instruction_data *data __rte_unused)
3302 {
3303         char *dst = tokens[1], *src = tokens[2];
3304         struct field *fdst, *fsrc;
3305         uint64_t src_val;
3306         uint32_t dst_struct_id = 0, src_struct_id = 0;
3307
3308         CHECK(n_tokens == 3, EINVAL);
3309
3310         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3311         CHECK(fdst, EINVAL);
3312         CHECK(!fdst->var_size, EINVAL);
3313
3314         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3315         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3316         if (fsrc) {
3317                 CHECK(!fsrc->var_size, EINVAL);
3318
3319                 instr->type = INSTR_ALU_XOR;
3320                 if (dst[0] != 'h' && src[0] == 'h')
3321                         instr->type = INSTR_ALU_XOR_MH;
3322                 if (dst[0] == 'h' && src[0] != 'h')
3323                         instr->type = INSTR_ALU_XOR_HM;
3324                 if (dst[0] == 'h' && src[0] == 'h')
3325                         instr->type = INSTR_ALU_XOR_HH;
3326
3327                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3328                 instr->alu.dst.n_bits = fdst->n_bits;
3329                 instr->alu.dst.offset = fdst->offset / 8;
3330                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3331                 instr->alu.src.n_bits = fsrc->n_bits;
3332                 instr->alu.src.offset = fsrc->offset / 8;
3333                 return 0;
3334         }
3335
3336         /* XOR_I. */
3337         src_val = strtoull(src, &src, 0);
3338         CHECK(!src[0], EINVAL);
3339
3340         if (dst[0] == 'h')
3341                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3342
3343         instr->type = INSTR_ALU_XOR_I;
3344         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3345         instr->alu.dst.n_bits = fdst->n_bits;
3346         instr->alu.dst.offset = fdst->offset / 8;
3347         instr->alu.src_val = src_val;
3348         return 0;
3349 }
3350
3351 static inline void
3352 instr_alu_add_exec(struct rte_swx_pipeline *p)
3353 {
3354         struct thread *t = &p->threads[p->thread_id];
3355         struct instruction *ip = t->ip;
3356
3357         TRACE("[Thread %2u] add\n", p->thread_id);
3358
3359         /* Structs. */
3360         ALU(t, ip, +);
3361
3362         /* Thread. */
3363         thread_ip_inc(p);
3364 }
3365
3366 static inline void
3367 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3368 {
3369         struct thread *t = &p->threads[p->thread_id];
3370         struct instruction *ip = t->ip;
3371
3372         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3373
3374         /* Structs. */
3375         ALU_MH(t, ip, +);
3376
3377         /* Thread. */
3378         thread_ip_inc(p);
3379 }
3380
3381 static inline void
3382 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3383 {
3384         struct thread *t = &p->threads[p->thread_id];
3385         struct instruction *ip = t->ip;
3386
3387         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3388
3389         /* Structs. */
3390         ALU_HM(t, ip, +);
3391
3392         /* Thread. */
3393         thread_ip_inc(p);
3394 }
3395
3396 static inline void
3397 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3398 {
3399         struct thread *t = &p->threads[p->thread_id];
3400         struct instruction *ip = t->ip;
3401
3402         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3403
3404         /* Structs. */
3405         ALU_HH(t, ip, +);
3406
3407         /* Thread. */
3408         thread_ip_inc(p);
3409 }
3410
3411 static inline void
3412 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3413 {
3414         struct thread *t = &p->threads[p->thread_id];
3415         struct instruction *ip = t->ip;
3416
3417         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3418
3419         /* Structs. */
3420         ALU_MI(t, ip, +);
3421
3422         /* Thread. */
3423         thread_ip_inc(p);
3424 }
3425
3426 static inline void
3427 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3428 {
3429         struct thread *t = &p->threads[p->thread_id];
3430         struct instruction *ip = t->ip;
3431
3432         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3433
3434         /* Structs. */
3435         ALU_HI(t, ip, +);
3436
3437         /* Thread. */
3438         thread_ip_inc(p);
3439 }
3440
3441 static inline void
3442 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3443 {
3444         struct thread *t = &p->threads[p->thread_id];
3445         struct instruction *ip = t->ip;
3446
3447         TRACE("[Thread %2u] sub\n", p->thread_id);
3448
3449         /* Structs. */
3450         ALU(t, ip, -);
3451
3452         /* Thread. */
3453         thread_ip_inc(p);
3454 }
3455
3456 static inline void
3457 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3458 {
3459         struct thread *t = &p->threads[p->thread_id];
3460         struct instruction *ip = t->ip;
3461
3462         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3463
3464         /* Structs. */
3465         ALU_MH(t, ip, -);
3466
3467         /* Thread. */
3468         thread_ip_inc(p);
3469 }
3470
3471 static inline void
3472 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3473 {
3474         struct thread *t = &p->threads[p->thread_id];
3475         struct instruction *ip = t->ip;
3476
3477         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3478
3479         /* Structs. */
3480         ALU_HM(t, ip, -);
3481
3482         /* Thread. */
3483         thread_ip_inc(p);
3484 }
3485
3486 static inline void
3487 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3488 {
3489         struct thread *t = &p->threads[p->thread_id];
3490         struct instruction *ip = t->ip;
3491
3492         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3493
3494         /* Structs. */
3495         ALU_HH(t, ip, -);
3496
3497         /* Thread. */
3498         thread_ip_inc(p);
3499 }
3500
3501 static inline void
3502 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3503 {
3504         struct thread *t = &p->threads[p->thread_id];
3505         struct instruction *ip = t->ip;
3506
3507         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3508
3509         /* Structs. */
3510         ALU_MI(t, ip, -);
3511
3512         /* Thread. */
3513         thread_ip_inc(p);
3514 }
3515
3516 static inline void
3517 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3518 {
3519         struct thread *t = &p->threads[p->thread_id];
3520         struct instruction *ip = t->ip;
3521
3522         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3523
3524         /* Structs. */
3525         ALU_HI(t, ip, -);
3526
3527         /* Thread. */
3528         thread_ip_inc(p);
3529 }
3530
3531 static inline void
3532 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3533 {
3534         struct thread *t = &p->threads[p->thread_id];
3535         struct instruction *ip = t->ip;
3536
3537         TRACE("[Thread %2u] shl\n", p->thread_id);
3538
3539         /* Structs. */
3540         ALU(t, ip, <<);
3541
3542         /* Thread. */
3543         thread_ip_inc(p);
3544 }
3545
3546 static inline void
3547 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3548 {
3549         struct thread *t = &p->threads[p->thread_id];
3550         struct instruction *ip = t->ip;
3551
3552         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3553
3554         /* Structs. */
3555         ALU_MH(t, ip, <<);
3556
3557         /* Thread. */
3558         thread_ip_inc(p);
3559 }
3560
3561 static inline void
3562 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3563 {
3564         struct thread *t = &p->threads[p->thread_id];
3565         struct instruction *ip = t->ip;
3566
3567         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3568
3569         /* Structs. */
3570         ALU_HM(t, ip, <<);
3571
3572         /* Thread. */
3573         thread_ip_inc(p);
3574 }
3575
3576 static inline void
3577 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3578 {
3579         struct thread *t = &p->threads[p->thread_id];
3580         struct instruction *ip = t->ip;
3581
3582         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3583
3584         /* Structs. */
3585         ALU_HH(t, ip, <<);
3586
3587         /* Thread. */
3588         thread_ip_inc(p);
3589 }
3590
3591 static inline void
3592 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3593 {
3594         struct thread *t = &p->threads[p->thread_id];
3595         struct instruction *ip = t->ip;
3596
3597         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3598
3599         /* Structs. */
3600         ALU_MI(t, ip, <<);
3601
3602         /* Thread. */
3603         thread_ip_inc(p);
3604 }
3605
3606 static inline void
3607 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3608 {
3609         struct thread *t = &p->threads[p->thread_id];
3610         struct instruction *ip = t->ip;
3611
3612         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3613
3614         /* Structs. */
3615         ALU_HI(t, ip, <<);
3616
3617         /* Thread. */
3618         thread_ip_inc(p);
3619 }
3620
3621 static inline void
3622 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3623 {
3624         struct thread *t = &p->threads[p->thread_id];
3625         struct instruction *ip = t->ip;
3626
3627         TRACE("[Thread %2u] shr\n", p->thread_id);
3628
3629         /* Structs. */
3630         ALU(t, ip, >>);
3631
3632         /* Thread. */
3633         thread_ip_inc(p);
3634 }
3635
3636 static inline void
3637 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3638 {
3639         struct thread *t = &p->threads[p->thread_id];
3640         struct instruction *ip = t->ip;
3641
3642         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3643
3644         /* Structs. */
3645         ALU_MH(t, ip, >>);
3646
3647         /* Thread. */
3648         thread_ip_inc(p);
3649 }
3650
3651 static inline void
3652 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3653 {
3654         struct thread *t = &p->threads[p->thread_id];
3655         struct instruction *ip = t->ip;
3656
3657         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3658
3659         /* Structs. */
3660         ALU_HM(t, ip, >>);
3661
3662         /* Thread. */
3663         thread_ip_inc(p);
3664 }
3665
3666 static inline void
3667 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3668 {
3669         struct thread *t = &p->threads[p->thread_id];
3670         struct instruction *ip = t->ip;
3671
3672         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3673
3674         /* Structs. */
3675         ALU_HH(t, ip, >>);
3676
3677         /* Thread. */
3678         thread_ip_inc(p);
3679 }
3680
3681 static inline void
3682 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3683 {
3684         struct thread *t = &p->threads[p->thread_id];
3685         struct instruction *ip = t->ip;
3686
3687         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3688
3689         /* Structs. */
3690         ALU_MI(t, ip, >>);
3691
3692         /* Thread. */
3693         thread_ip_inc(p);
3694 }
3695
3696 static inline void
3697 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3698 {
3699         struct thread *t = &p->threads[p->thread_id];
3700         struct instruction *ip = t->ip;
3701
3702         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3703
3704         /* Structs. */
3705         ALU_HI(t, ip, >>);
3706
3707         /* Thread. */
3708         thread_ip_inc(p);
3709 }
3710
3711 static inline void
3712 instr_alu_and_exec(struct rte_swx_pipeline *p)
3713 {
3714         struct thread *t = &p->threads[p->thread_id];
3715         struct instruction *ip = t->ip;
3716
3717         TRACE("[Thread %2u] and\n", p->thread_id);
3718
3719         /* Structs. */
3720         ALU(t, ip, &);
3721
3722         /* Thread. */
3723         thread_ip_inc(p);
3724 }
3725
3726 static inline void
3727 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3728 {
3729         struct thread *t = &p->threads[p->thread_id];
3730         struct instruction *ip = t->ip;
3731
3732         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3733
3734         /* Structs. */
3735         ALU_MH(t, ip, &);
3736
3737         /* Thread. */
3738         thread_ip_inc(p);
3739 }
3740
3741 static inline void
3742 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3743 {
3744         struct thread *t = &p->threads[p->thread_id];
3745         struct instruction *ip = t->ip;
3746
3747         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3748
3749         /* Structs. */
3750         ALU_HM_FAST(t, ip, &);
3751
3752         /* Thread. */
3753         thread_ip_inc(p);
3754 }
3755
3756 static inline void
3757 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3758 {
3759         struct thread *t = &p->threads[p->thread_id];
3760         struct instruction *ip = t->ip;
3761
3762         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3763
3764         /* Structs. */
3765         ALU_HH_FAST(t, ip, &);
3766
3767         /* Thread. */
3768         thread_ip_inc(p);
3769 }
3770
3771 static inline void
3772 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3773 {
3774         struct thread *t = &p->threads[p->thread_id];
3775         struct instruction *ip = t->ip;
3776
3777         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3778
3779         /* Structs. */
3780         ALU_I(t, ip, &);
3781
3782         /* Thread. */
3783         thread_ip_inc(p);
3784 }
3785
3786 static inline void
3787 instr_alu_or_exec(struct rte_swx_pipeline *p)
3788 {
3789         struct thread *t = &p->threads[p->thread_id];
3790         struct instruction *ip = t->ip;
3791
3792         TRACE("[Thread %2u] or\n", p->thread_id);
3793
3794         /* Structs. */
3795         ALU(t, ip, |);
3796
3797         /* Thread. */
3798         thread_ip_inc(p);
3799 }
3800
3801 static inline void
3802 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3803 {
3804         struct thread *t = &p->threads[p->thread_id];
3805         struct instruction *ip = t->ip;
3806
3807         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3808
3809         /* Structs. */
3810         ALU_MH(t, ip, |);
3811
3812         /* Thread. */
3813         thread_ip_inc(p);
3814 }
3815
3816 static inline void
3817 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3818 {
3819         struct thread *t = &p->threads[p->thread_id];
3820         struct instruction *ip = t->ip;
3821
3822         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3823
3824         /* Structs. */
3825         ALU_HM_FAST(t, ip, |);
3826
3827         /* Thread. */
3828         thread_ip_inc(p);
3829 }
3830
3831 static inline void
3832 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3833 {
3834         struct thread *t = &p->threads[p->thread_id];
3835         struct instruction *ip = t->ip;
3836
3837         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3838
3839         /* Structs. */
3840         ALU_HH_FAST(t, ip, |);
3841
3842         /* Thread. */
3843         thread_ip_inc(p);
3844 }
3845
3846 static inline void
3847 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3848 {
3849         struct thread *t = &p->threads[p->thread_id];
3850         struct instruction *ip = t->ip;
3851
3852         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3853
3854         /* Structs. */
3855         ALU_I(t, ip, |);
3856
3857         /* Thread. */
3858         thread_ip_inc(p);
3859 }
3860
3861 static inline void
3862 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3863 {
3864         struct thread *t = &p->threads[p->thread_id];
3865         struct instruction *ip = t->ip;
3866
3867         TRACE("[Thread %2u] xor\n", p->thread_id);
3868
3869         /* Structs. */
3870         ALU(t, ip, ^);
3871
3872         /* Thread. */
3873         thread_ip_inc(p);
3874 }
3875
3876 static inline void
3877 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3878 {
3879         struct thread *t = &p->threads[p->thread_id];
3880         struct instruction *ip = t->ip;
3881
3882         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3883
3884         /* Structs. */
3885         ALU_MH(t, ip, ^);
3886
3887         /* Thread. */
3888         thread_ip_inc(p);
3889 }
3890
3891 static inline void
3892 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3893 {
3894         struct thread *t = &p->threads[p->thread_id];
3895         struct instruction *ip = t->ip;
3896
3897         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3898
3899         /* Structs. */
3900         ALU_HM_FAST(t, ip, ^);
3901
3902         /* Thread. */
3903         thread_ip_inc(p);
3904 }
3905
3906 static inline void
3907 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3908 {
3909         struct thread *t = &p->threads[p->thread_id];
3910         struct instruction *ip = t->ip;
3911
3912         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3913
3914         /* Structs. */
3915         ALU_HH_FAST(t, ip, ^);
3916
3917         /* Thread. */
3918         thread_ip_inc(p);
3919 }
3920
3921 static inline void
3922 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3923 {
3924         struct thread *t = &p->threads[p->thread_id];
3925         struct instruction *ip = t->ip;
3926
3927         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3928
3929         /* Structs. */
3930         ALU_I(t, ip, ^);
3931
3932         /* Thread. */
3933         thread_ip_inc(p);
3934 }
3935
3936 static inline void
3937 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3938 {
3939         struct thread *t = &p->threads[p->thread_id];
3940         struct instruction *ip = t->ip;
3941         uint8_t *dst_struct, *src_struct;
3942         uint16_t *dst16_ptr, dst;
3943         uint64_t *src64_ptr, src64, src64_mask, src;
3944         uint64_t r;
3945
3946         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3947
3948         /* Structs. */
3949         dst_struct = t->structs[ip->alu.dst.struct_id];
3950         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3951         dst = *dst16_ptr;
3952
3953         src_struct = t->structs[ip->alu.src.struct_id];
3954         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3955         src64 = *src64_ptr;
3956         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3957         src = src64 & src64_mask;
3958
3959         r = dst;
3960         r = ~r & 0xFFFF;
3961
3962         /* The first input (r) is a 16-bit number. The second and the third
3963          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3964          * three numbers (output r) is a 34-bit number.
3965          */
3966         r += (src >> 32) + (src & 0xFFFFFFFF);
3967
3968         /* The first input is a 16-bit number. The second input is an 18-bit
3969          * number. In the worst case scenario, the sum of the two numbers is a
3970          * 19-bit number.
3971          */
3972         r = (r & 0xFFFF) + (r >> 16);
3973
3974         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3975          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3976          */
3977         r = (r & 0xFFFF) + (r >> 16);
3978
3979         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3980          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3981          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3982          * therefore the output r is always a 16-bit number.
3983          */
3984         r = (r & 0xFFFF) + (r >> 16);
3985
3986         r = ~r & 0xFFFF;
3987         r = r ? r : 0xFFFF;
3988
3989         *dst16_ptr = (uint16_t)r;
3990
3991         /* Thread. */
3992         thread_ip_inc(p);
3993 }
3994
3995 static inline void
3996 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3997 {
3998         struct thread *t = &p->threads[p->thread_id];
3999         struct instruction *ip = t->ip;
4000         uint8_t *dst_struct, *src_struct;
4001         uint16_t *dst16_ptr, dst;
4002         uint64_t *src64_ptr, src64, src64_mask, src;
4003         uint64_t r;
4004
4005         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4006
4007         /* Structs. */
4008         dst_struct = t->structs[ip->alu.dst.struct_id];
4009         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4010         dst = *dst16_ptr;
4011
4012         src_struct = t->structs[ip->alu.src.struct_id];
4013         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4014         src64 = *src64_ptr;
4015         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4016         src = src64 & src64_mask;
4017
4018         r = dst;
4019         r = ~r & 0xFFFF;
4020
4021         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4022          * the following sequence of operations in 2's complement arithmetic:
4023          *    a '- b = (a - b) % 0xFFFF.
4024          *
4025          * In order to prevent an underflow for the below subtraction, in which
4026          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4027          * minuend), we first add a multiple of the 0xFFFF modulus to the
4028          * minuend. The number we add to the minuend needs to be a 34-bit number
4029          * or higher, so for readability reasons we picked the 36-bit multiple.
4030          * We are effectively turning the 16-bit minuend into a 36-bit number:
4031          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4032          */
4033         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4034
4035         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4036          * result (the output r) is a 36-bit number.
4037          */
4038         r -= (src >> 32) + (src & 0xFFFFFFFF);
4039
4040         /* The first input is a 16-bit number. The second input is a 20-bit
4041          * number. Their sum is a 21-bit number.
4042          */
4043         r = (r & 0xFFFF) + (r >> 16);
4044
4045         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4046          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4047          */
4048         r = (r & 0xFFFF) + (r >> 16);
4049
4050         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4051          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4052          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4053          * generated, therefore the output r is always a 16-bit number.
4054          */
4055         r = (r & 0xFFFF) + (r >> 16);
4056
4057         r = ~r & 0xFFFF;
4058         r = r ? r : 0xFFFF;
4059
4060         *dst16_ptr = (uint16_t)r;
4061
4062         /* Thread. */
4063         thread_ip_inc(p);
4064 }
4065
4066 static inline void
4067 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4068 {
4069         struct thread *t = &p->threads[p->thread_id];
4070         struct instruction *ip = t->ip;
4071         uint8_t *dst_struct, *src_struct;
4072         uint16_t *dst16_ptr;
4073         uint32_t *src32_ptr;
4074         uint64_t r0, r1;
4075
4076         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4077
4078         /* Structs. */
4079         dst_struct = t->structs[ip->alu.dst.struct_id];
4080         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4081
4082         src_struct = t->structs[ip->alu.src.struct_id];
4083         src32_ptr = (uint32_t *)&src_struct[0];
4084
4085         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4086         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4087         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4088         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4089         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4090
4091         /* The first input is a 16-bit number. The second input is a 19-bit
4092          * number. Their sum is a 20-bit number.
4093          */
4094         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4095
4096         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4097          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4098          */
4099         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4100
4101         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4102          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4103          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4104          * generated, therefore the output r is always a 16-bit number.
4105          */
4106         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4107
4108         r0 = ~r0 & 0xFFFF;
4109         r0 = r0 ? r0 : 0xFFFF;
4110
4111         *dst16_ptr = (uint16_t)r0;
4112
4113         /* Thread. */
4114         thread_ip_inc(p);
4115 }
4116
4117 static inline void
4118 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4119 {
4120         struct thread *t = &p->threads[p->thread_id];
4121         struct instruction *ip = t->ip;
4122         uint8_t *dst_struct, *src_struct;
4123         uint16_t *dst16_ptr;
4124         uint32_t *src32_ptr;
4125         uint64_t r = 0;
4126         uint32_t i;
4127
4128         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4129
4130         /* Structs. */
4131         dst_struct = t->structs[ip->alu.dst.struct_id];
4132         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4133
4134         src_struct = t->structs[ip->alu.src.struct_id];
4135         src32_ptr = (uint32_t *)&src_struct[0];
4136
4137         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4138          * Therefore, in the worst case scenario, a 35-bit number is added to a
4139          * 16-bit number (the input r), so the output r is 36-bit number.
4140          */
4141         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4142                 r += *src32_ptr;
4143
4144         /* The first input is a 16-bit number. The second input is a 20-bit
4145          * number. Their sum is a 21-bit number.
4146          */
4147         r = (r & 0xFFFF) + (r >> 16);
4148
4149         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4150          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4151          */
4152         r = (r & 0xFFFF) + (r >> 16);
4153
4154         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4155          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4156          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4157          * generated, therefore the output r is always a 16-bit number.
4158          */
4159         r = (r & 0xFFFF) + (r >> 16);
4160
4161         r = ~r & 0xFFFF;
4162         r = r ? r : 0xFFFF;
4163
4164         *dst16_ptr = (uint16_t)r;
4165
4166         /* Thread. */
4167         thread_ip_inc(p);
4168 }
4169
4170 /*
4171  * Register array.
4172  */
4173 static struct regarray *
4174 regarray_find(struct rte_swx_pipeline *p, const char *name);
4175
4176 static int
4177 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4178                       struct action *action,
4179                       char **tokens,
4180                       int n_tokens,
4181                       struct instruction *instr,
4182                       struct instruction_data *data __rte_unused)
4183 {
4184         char *regarray = tokens[1], *idx = tokens[2];
4185         struct regarray *r;
4186         struct field *fidx;
4187         uint32_t idx_struct_id, idx_val;
4188
4189         CHECK(n_tokens == 3, EINVAL);
4190
4191         r = regarray_find(p, regarray);
4192         CHECK(r, EINVAL);
4193
4194         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4195         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4196         if (fidx) {
4197                 CHECK(!fidx->var_size, EINVAL);
4198
4199                 instr->type = INSTR_REGPREFETCH_RM;
4200                 if (idx[0] == 'h')
4201                         instr->type = INSTR_REGPREFETCH_RH;
4202
4203                 instr->regarray.regarray_id = r->id;
4204                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4205                 instr->regarray.idx.n_bits = fidx->n_bits;
4206                 instr->regarray.idx.offset = fidx->offset / 8;
4207                 instr->regarray.dstsrc_val = 0; /* Unused. */
4208                 return 0;
4209         }
4210
4211         /* REGPREFETCH_RI. */
4212         idx_val = strtoul(idx, &idx, 0);
4213         CHECK(!idx[0], EINVAL);
4214
4215         instr->type = INSTR_REGPREFETCH_RI;
4216         instr->regarray.regarray_id = r->id;
4217         instr->regarray.idx_val = idx_val;
4218         instr->regarray.dstsrc_val = 0; /* Unused. */
4219         return 0;
4220 }
4221
4222 static int
4223 instr_regrd_translate(struct rte_swx_pipeline *p,
4224                       struct action *action,
4225                       char **tokens,
4226                       int n_tokens,
4227                       struct instruction *instr,
4228                       struct instruction_data *data __rte_unused)
4229 {
4230         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4231         struct regarray *r;
4232         struct field *fdst, *fidx;
4233         uint32_t dst_struct_id, idx_struct_id, idx_val;
4234
4235         CHECK(n_tokens == 4, EINVAL);
4236
4237         r = regarray_find(p, regarray);
4238         CHECK(r, EINVAL);
4239
4240         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4241         CHECK(fdst, EINVAL);
4242         CHECK(!fdst->var_size, EINVAL);
4243
4244         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4245         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4246         if (fidx) {
4247                 CHECK(!fidx->var_size, EINVAL);
4248
4249                 instr->type = INSTR_REGRD_MRM;
4250                 if (dst[0] == 'h' && idx[0] != 'h')
4251                         instr->type = INSTR_REGRD_HRM;
4252                 if (dst[0] != 'h' && idx[0] == 'h')
4253                         instr->type = INSTR_REGRD_MRH;
4254                 if (dst[0] == 'h' && idx[0] == 'h')
4255                         instr->type = INSTR_REGRD_HRH;
4256
4257                 instr->regarray.regarray_id = r->id;
4258                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4259                 instr->regarray.idx.n_bits = fidx->n_bits;
4260                 instr->regarray.idx.offset = fidx->offset / 8;
4261                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4262                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4263                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4264                 return 0;
4265         }
4266
4267         /* REGRD_MRI, REGRD_HRI. */
4268         idx_val = strtoul(idx, &idx, 0);
4269         CHECK(!idx[0], EINVAL);
4270
4271         instr->type = INSTR_REGRD_MRI;
4272         if (dst[0] == 'h')
4273                 instr->type = INSTR_REGRD_HRI;
4274
4275         instr->regarray.regarray_id = r->id;
4276         instr->regarray.idx_val = idx_val;
4277         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4278         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4279         instr->regarray.dstsrc.offset = fdst->offset / 8;
4280         return 0;
4281 }
4282
4283 static int
4284 instr_regwr_translate(struct rte_swx_pipeline *p,
4285                       struct action *action,
4286                       char **tokens,
4287                       int n_tokens,
4288                       struct instruction *instr,
4289                       struct instruction_data *data __rte_unused)
4290 {
4291         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4292         struct regarray *r;
4293         struct field *fidx, *fsrc;
4294         uint64_t src_val;
4295         uint32_t idx_struct_id, idx_val, src_struct_id;
4296
4297         CHECK(n_tokens == 4, EINVAL);
4298
4299         r = regarray_find(p, regarray);
4300         CHECK(r, EINVAL);
4301
4302         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4303         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4304         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4305         if (fidx && fsrc) {
4306                 CHECK(!fidx->var_size, EINVAL);
4307                 CHECK(!fsrc->var_size, EINVAL);
4308
4309                 instr->type = INSTR_REGWR_RMM;
4310                 if (idx[0] == 'h' && src[0] != 'h')
4311                         instr->type = INSTR_REGWR_RHM;
4312                 if (idx[0] != 'h' && src[0] == 'h')
4313                         instr->type = INSTR_REGWR_RMH;
4314                 if (idx[0] == 'h' && src[0] == 'h')
4315                         instr->type = INSTR_REGWR_RHH;
4316
4317                 instr->regarray.regarray_id = r->id;
4318                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4319                 instr->regarray.idx.n_bits = fidx->n_bits;
4320                 instr->regarray.idx.offset = fidx->offset / 8;
4321                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4322                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4323                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4324                 return 0;
4325         }
4326
4327         /* REGWR_RHI, REGWR_RMI. */
4328         if (fidx && !fsrc) {
4329                 CHECK(!fidx->var_size, EINVAL);
4330
4331                 src_val = strtoull(src, &src, 0);
4332                 CHECK(!src[0], EINVAL);
4333
4334                 instr->type = INSTR_REGWR_RMI;
4335                 if (idx[0] == 'h')
4336                         instr->type = INSTR_REGWR_RHI;
4337
4338                 instr->regarray.regarray_id = r->id;
4339                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4340                 instr->regarray.idx.n_bits = fidx->n_bits;
4341                 instr->regarray.idx.offset = fidx->offset / 8;
4342                 instr->regarray.dstsrc_val = src_val;
4343                 return 0;
4344         }
4345
4346         /* REGWR_RIH, REGWR_RIM. */
4347         if (!fidx && fsrc) {
4348                 idx_val = strtoul(idx, &idx, 0);
4349                 CHECK(!idx[0], EINVAL);
4350
4351                 CHECK(!fsrc->var_size, EINVAL);
4352
4353                 instr->type = INSTR_REGWR_RIM;
4354                 if (src[0] == 'h')
4355                         instr->type = INSTR_REGWR_RIH;
4356
4357                 instr->regarray.regarray_id = r->id;
4358                 instr->regarray.idx_val = idx_val;
4359                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4360                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4361                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4362                 return 0;
4363         }
4364
4365         /* REGWR_RII. */
4366         src_val = strtoull(src, &src, 0);
4367         CHECK(!src[0], EINVAL);
4368
4369         idx_val = strtoul(idx, &idx, 0);
4370         CHECK(!idx[0], EINVAL);
4371
4372         instr->type = INSTR_REGWR_RII;
4373         instr->regarray.idx_val = idx_val;
4374         instr->regarray.dstsrc_val = src_val;
4375
4376         return 0;
4377 }
4378
4379 static int
4380 instr_regadd_translate(struct rte_swx_pipeline *p,
4381                        struct action *action,
4382                        char **tokens,
4383                        int n_tokens,
4384                        struct instruction *instr,
4385                        struct instruction_data *data __rte_unused)
4386 {
4387         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4388         struct regarray *r;
4389         struct field *fidx, *fsrc;
4390         uint64_t src_val;
4391         uint32_t idx_struct_id, idx_val, src_struct_id;
4392
4393         CHECK(n_tokens == 4, EINVAL);
4394
4395         r = regarray_find(p, regarray);
4396         CHECK(r, EINVAL);
4397
4398         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4399         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4400         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4401         if (fidx && fsrc) {
4402                 CHECK(!fidx->var_size, EINVAL);
4403                 CHECK(!fsrc->var_size, EINVAL);
4404
4405                 instr->type = INSTR_REGADD_RMM;
4406                 if (idx[0] == 'h' && src[0] != 'h')
4407                         instr->type = INSTR_REGADD_RHM;
4408                 if (idx[0] != 'h' && src[0] == 'h')
4409                         instr->type = INSTR_REGADD_RMH;
4410                 if (idx[0] == 'h' && src[0] == 'h')
4411                         instr->type = INSTR_REGADD_RHH;
4412
4413                 instr->regarray.regarray_id = r->id;
4414                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4415                 instr->regarray.idx.n_bits = fidx->n_bits;
4416                 instr->regarray.idx.offset = fidx->offset / 8;
4417                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4418                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4419                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4420                 return 0;
4421         }
4422
4423         /* REGADD_RHI, REGADD_RMI. */
4424         if (fidx && !fsrc) {
4425                 CHECK(!fidx->var_size, EINVAL);
4426
4427                 src_val = strtoull(src, &src, 0);
4428                 CHECK(!src[0], EINVAL);
4429
4430                 instr->type = INSTR_REGADD_RMI;
4431                 if (idx[0] == 'h')
4432                         instr->type = INSTR_REGADD_RHI;
4433
4434                 instr->regarray.regarray_id = r->id;
4435                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4436                 instr->regarray.idx.n_bits = fidx->n_bits;
4437                 instr->regarray.idx.offset = fidx->offset / 8;
4438                 instr->regarray.dstsrc_val = src_val;
4439                 return 0;
4440         }
4441
4442         /* REGADD_RIH, REGADD_RIM. */
4443         if (!fidx && fsrc) {
4444                 idx_val = strtoul(idx, &idx, 0);
4445                 CHECK(!idx[0], EINVAL);
4446
4447                 CHECK(!fsrc->var_size, EINVAL);
4448
4449                 instr->type = INSTR_REGADD_RIM;
4450                 if (src[0] == 'h')
4451                         instr->type = INSTR_REGADD_RIH;
4452
4453                 instr->regarray.regarray_id = r->id;
4454                 instr->regarray.idx_val = idx_val;
4455                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4456                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4457                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4458                 return 0;
4459         }
4460
4461         /* REGADD_RII. */
4462         src_val = strtoull(src, &src, 0);
4463         CHECK(!src[0], EINVAL);
4464
4465         idx_val = strtoul(idx, &idx, 0);
4466         CHECK(!idx[0], EINVAL);
4467
4468         instr->type = INSTR_REGADD_RII;
4469         instr->regarray.idx_val = idx_val;
4470         instr->regarray.dstsrc_val = src_val;
4471         return 0;
4472 }
4473
4474 static inline uint64_t *
4475 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4476 {
4477         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4478         return r->regarray;
4479 }
4480
4481 static inline uint64_t
4482 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4483 {
4484         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4485
4486         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4487         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4488         uint64_t idx64 = *idx64_ptr;
4489         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4490         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4491
4492         return idx;
4493 }
4494
4495 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4496
4497 static inline uint64_t
4498 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4499 {
4500         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4501
4502         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4503         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4504         uint64_t idx64 = *idx64_ptr;
4505         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4506
4507         return idx;
4508 }
4509
4510 #else
4511
4512 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4513
4514 #endif
4515
4516 static inline uint64_t
4517 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4518 {
4519         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4520
4521         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4522
4523         return idx;
4524 }
4525
4526 static inline uint64_t
4527 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4528 {
4529         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4530         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4531         uint64_t src64 = *src64_ptr;
4532         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4533         uint64_t src = src64 & src64_mask;
4534
4535         return src;
4536 }
4537
4538 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4539
4540 static inline uint64_t
4541 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4542 {
4543         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4544         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4545         uint64_t src64 = *src64_ptr;
4546         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4547
4548         return src;
4549 }
4550
4551 #else
4552
4553 #define instr_regarray_src_nbo instr_regarray_src_hbo
4554
4555 #endif
4556
4557 static inline void
4558 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4559 {
4560         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4561         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4562         uint64_t dst64 = *dst64_ptr;
4563         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4564
4565         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4566
4567 }
4568
4569 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4570
4571 static inline void
4572 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4573 {
4574         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4575         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4576         uint64_t dst64 = *dst64_ptr;
4577         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4578
4579         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4580         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4581 }
4582
4583 #else
4584
4585 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4586
4587 #endif
4588
4589 static inline void
4590 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4591 {
4592         struct thread *t = &p->threads[p->thread_id];
4593         struct instruction *ip = t->ip;
4594         uint64_t *regarray, idx;
4595
4596         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4597
4598         /* Structs. */
4599         regarray = instr_regarray_regarray(p, ip);
4600         idx = instr_regarray_idx_nbo(p, t, ip);
4601         rte_prefetch0(&regarray[idx]);
4602
4603         /* Thread. */
4604         thread_ip_inc(p);
4605 }
4606
4607 static inline void
4608 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4609 {
4610         struct thread *t = &p->threads[p->thread_id];
4611         struct instruction *ip = t->ip;
4612         uint64_t *regarray, idx;
4613
4614         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4615
4616         /* Structs. */
4617         regarray = instr_regarray_regarray(p, ip);
4618         idx = instr_regarray_idx_hbo(p, t, ip);
4619         rte_prefetch0(&regarray[idx]);
4620
4621         /* Thread. */
4622         thread_ip_inc(p);
4623 }
4624
4625 static inline void
4626 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4627 {
4628         struct thread *t = &p->threads[p->thread_id];
4629         struct instruction *ip = t->ip;
4630         uint64_t *regarray, idx;
4631
4632         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4633
4634         /* Structs. */
4635         regarray = instr_regarray_regarray(p, ip);
4636         idx = instr_regarray_idx_imm(p, ip);
4637         rte_prefetch0(&regarray[idx]);
4638
4639         /* Thread. */
4640         thread_ip_inc(p);
4641 }
4642
4643 static inline void
4644 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4645 {
4646         struct thread *t = &p->threads[p->thread_id];
4647         struct instruction *ip = t->ip;
4648         uint64_t *regarray, idx;
4649
4650         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4651
4652         /* Structs. */
4653         regarray = instr_regarray_regarray(p, ip);
4654         idx = instr_regarray_idx_nbo(p, t, ip);
4655         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4656
4657         /* Thread. */
4658         thread_ip_inc(p);
4659 }
4660
4661 static inline void
4662 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4663 {
4664         struct thread *t = &p->threads[p->thread_id];
4665         struct instruction *ip = t->ip;
4666         uint64_t *regarray, idx;
4667
4668         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4669
4670         /* Structs. */
4671         regarray = instr_regarray_regarray(p, ip);
4672         idx = instr_regarray_idx_hbo(p, t, ip);
4673         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4674
4675         /* Thread. */
4676         thread_ip_inc(p);
4677 }
4678
4679 static inline void
4680 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4681 {
4682         struct thread *t = &p->threads[p->thread_id];
4683         struct instruction *ip = t->ip;
4684         uint64_t *regarray, idx;
4685
4686         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4687
4688         /* Structs. */
4689         regarray = instr_regarray_regarray(p, ip);
4690         idx = instr_regarray_idx_nbo(p, t, ip);
4691         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4692
4693         /* Thread. */
4694         thread_ip_inc(p);
4695 }
4696
4697 static inline void
4698 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4699 {
4700         struct thread *t = &p->threads[p->thread_id];
4701         struct instruction *ip = t->ip;
4702         uint64_t *regarray, idx;
4703
4704         /* Structs. */
4705         regarray = instr_regarray_regarray(p, ip);
4706         idx = instr_regarray_idx_hbo(p, t, ip);
4707         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4708
4709         /* Thread. */
4710         thread_ip_inc(p);
4711 }
4712
4713 static inline void
4714 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4715 {
4716         struct thread *t = &p->threads[p->thread_id];
4717         struct instruction *ip = t->ip;
4718         uint64_t *regarray, idx;
4719
4720         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4721
4722         /* Structs. */
4723         regarray = instr_regarray_regarray(p, ip);
4724         idx = instr_regarray_idx_imm(p, ip);
4725         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4726
4727         /* Thread. */
4728         thread_ip_inc(p);
4729 }
4730
4731 static inline void
4732 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4733 {
4734         struct thread *t = &p->threads[p->thread_id];
4735         struct instruction *ip = t->ip;
4736         uint64_t *regarray, idx;
4737
4738         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4739
4740         /* Structs. */
4741         regarray = instr_regarray_regarray(p, ip);
4742         idx = instr_regarray_idx_imm(p, ip);
4743         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4744
4745         /* Thread. */
4746         thread_ip_inc(p);
4747 }
4748
4749 static inline void
4750 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4751 {
4752         struct thread *t = &p->threads[p->thread_id];
4753         struct instruction *ip = t->ip;
4754         uint64_t *regarray, idx, src;
4755
4756         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4757
4758         /* Structs. */
4759         regarray = instr_regarray_regarray(p, ip);
4760         idx = instr_regarray_idx_nbo(p, t, ip);
4761         src = instr_regarray_src_nbo(t, ip);
4762         regarray[idx] = src;
4763
4764         /* Thread. */
4765         thread_ip_inc(p);
4766 }
4767
4768 static inline void
4769 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4770 {
4771         struct thread *t = &p->threads[p->thread_id];
4772         struct instruction *ip = t->ip;
4773         uint64_t *regarray, idx, src;
4774
4775         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4776
4777         /* Structs. */
4778         regarray = instr_regarray_regarray(p, ip);
4779         idx = instr_regarray_idx_nbo(p, t, ip);
4780         src = instr_regarray_src_hbo(t, ip);
4781         regarray[idx] = src;
4782
4783         /* Thread. */
4784         thread_ip_inc(p);
4785 }
4786
4787 static inline void
4788 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4789 {
4790         struct thread *t = &p->threads[p->thread_id];
4791         struct instruction *ip = t->ip;
4792         uint64_t *regarray, idx, src;
4793
4794         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4795
4796         /* Structs. */
4797         regarray = instr_regarray_regarray(p, ip);
4798         idx = instr_regarray_idx_hbo(p, t, ip);
4799         src = instr_regarray_src_nbo(t, ip);
4800         regarray[idx] = src;
4801
4802         /* Thread. */
4803         thread_ip_inc(p);
4804 }
4805
4806 static inline void
4807 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4808 {
4809         struct thread *t = &p->threads[p->thread_id];
4810         struct instruction *ip = t->ip;
4811         uint64_t *regarray, idx, src;
4812
4813         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4814
4815         /* Structs. */
4816         regarray = instr_regarray_regarray(p, ip);
4817         idx = instr_regarray_idx_hbo(p, t, ip);
4818         src = instr_regarray_src_hbo(t, ip);
4819         regarray[idx] = src;
4820
4821         /* Thread. */
4822         thread_ip_inc(p);
4823 }
4824
4825 static inline void
4826 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4827 {
4828         struct thread *t = &p->threads[p->thread_id];
4829         struct instruction *ip = t->ip;
4830         uint64_t *regarray, idx, src;
4831
4832         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4833
4834         /* Structs. */
4835         regarray = instr_regarray_regarray(p, ip);
4836         idx = instr_regarray_idx_nbo(p, t, ip);
4837         src = ip->regarray.dstsrc_val;
4838         regarray[idx] = src;
4839
4840         /* Thread. */
4841         thread_ip_inc(p);
4842 }
4843
4844 static inline void
4845 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4846 {
4847         struct thread *t = &p->threads[p->thread_id];
4848         struct instruction *ip = t->ip;
4849         uint64_t *regarray, idx, src;
4850
4851         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4852
4853         /* Structs. */
4854         regarray = instr_regarray_regarray(p, ip);
4855         idx = instr_regarray_idx_hbo(p, t, ip);
4856         src = ip->regarray.dstsrc_val;
4857         regarray[idx] = src;
4858
4859         /* Thread. */
4860         thread_ip_inc(p);
4861 }
4862
4863 static inline void
4864 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4865 {
4866         struct thread *t = &p->threads[p->thread_id];
4867         struct instruction *ip = t->ip;
4868         uint64_t *regarray, idx, src;
4869
4870         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4871
4872         /* Structs. */
4873         regarray = instr_regarray_regarray(p, ip);
4874         idx = instr_regarray_idx_imm(p, ip);
4875         src = instr_regarray_src_nbo(t, ip);
4876         regarray[idx] = src;
4877
4878         /* Thread. */
4879         thread_ip_inc(p);
4880 }
4881
4882 static inline void
4883 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4884 {
4885         struct thread *t = &p->threads[p->thread_id];
4886         struct instruction *ip = t->ip;
4887         uint64_t *regarray, idx, src;
4888
4889         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4890
4891         /* Structs. */
4892         regarray = instr_regarray_regarray(p, ip);
4893         idx = instr_regarray_idx_imm(p, ip);
4894         src = instr_regarray_src_hbo(t, ip);
4895         regarray[idx] = src;
4896
4897         /* Thread. */
4898         thread_ip_inc(p);
4899 }
4900
4901 static inline void
4902 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4903 {
4904         struct thread *t = &p->threads[p->thread_id];
4905         struct instruction *ip = t->ip;
4906         uint64_t *regarray, idx, src;
4907
4908         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4909
4910         /* Structs. */
4911         regarray = instr_regarray_regarray(p, ip);
4912         idx = instr_regarray_idx_imm(p, ip);
4913         src = ip->regarray.dstsrc_val;
4914         regarray[idx] = src;
4915
4916         /* Thread. */
4917         thread_ip_inc(p);
4918 }
4919
4920 static inline void
4921 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4922 {
4923         struct thread *t = &p->threads[p->thread_id];
4924         struct instruction *ip = t->ip;
4925         uint64_t *regarray, idx, src;
4926
4927         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4928
4929         /* Structs. */
4930         regarray = instr_regarray_regarray(p, ip);
4931         idx = instr_regarray_idx_nbo(p, t, ip);
4932         src = instr_regarray_src_nbo(t, ip);
4933         regarray[idx] += src;
4934
4935         /* Thread. */
4936         thread_ip_inc(p);
4937 }
4938
4939 static inline void
4940 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4941 {
4942         struct thread *t = &p->threads[p->thread_id];
4943         struct instruction *ip = t->ip;
4944         uint64_t *regarray, idx, src;
4945
4946         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4947
4948         /* Structs. */
4949         regarray = instr_regarray_regarray(p, ip);
4950         idx = instr_regarray_idx_nbo(p, t, ip);
4951         src = instr_regarray_src_hbo(t, ip);
4952         regarray[idx] += src;
4953
4954         /* Thread. */
4955         thread_ip_inc(p);
4956 }
4957
4958 static inline void
4959 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4960 {
4961         struct thread *t = &p->threads[p->thread_id];
4962         struct instruction *ip = t->ip;
4963         uint64_t *regarray, idx, src;
4964
4965         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4966
4967         /* Structs. */
4968         regarray = instr_regarray_regarray(p, ip);
4969         idx = instr_regarray_idx_hbo(p, t, ip);
4970         src = instr_regarray_src_nbo(t, ip);
4971         regarray[idx] += src;
4972
4973         /* Thread. */
4974         thread_ip_inc(p);
4975 }
4976
4977 static inline void
4978 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4979 {
4980         struct thread *t = &p->threads[p->thread_id];
4981         struct instruction *ip = t->ip;
4982         uint64_t *regarray, idx, src;
4983
4984         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4985
4986         /* Structs. */
4987         regarray = instr_regarray_regarray(p, ip);
4988         idx = instr_regarray_idx_hbo(p, t, ip);
4989         src = instr_regarray_src_hbo(t, ip);
4990         regarray[idx] += src;
4991
4992         /* Thread. */
4993         thread_ip_inc(p);
4994 }
4995
4996 static inline void
4997 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4998 {
4999         struct thread *t = &p->threads[p->thread_id];
5000         struct instruction *ip = t->ip;
5001         uint64_t *regarray, idx, src;
5002
5003         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5004
5005         /* Structs. */
5006         regarray = instr_regarray_regarray(p, ip);
5007         idx = instr_regarray_idx_nbo(p, t, ip);
5008         src = ip->regarray.dstsrc_val;
5009         regarray[idx] += src;
5010
5011         /* Thread. */
5012         thread_ip_inc(p);
5013 }
5014
5015 static inline void
5016 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5017 {
5018         struct thread *t = &p->threads[p->thread_id];
5019         struct instruction *ip = t->ip;
5020         uint64_t *regarray, idx, src;
5021
5022         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5023
5024         /* Structs. */
5025         regarray = instr_regarray_regarray(p, ip);
5026         idx = instr_regarray_idx_hbo(p, t, ip);
5027         src = ip->regarray.dstsrc_val;
5028         regarray[idx] += src;
5029
5030         /* Thread. */
5031         thread_ip_inc(p);
5032 }
5033
5034 static inline void
5035 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5036 {
5037         struct thread *t = &p->threads[p->thread_id];
5038         struct instruction *ip = t->ip;
5039         uint64_t *regarray, idx, src;
5040
5041         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5042
5043         /* Structs. */
5044         regarray = instr_regarray_regarray(p, ip);
5045         idx = instr_regarray_idx_imm(p, ip);
5046         src = instr_regarray_src_nbo(t, ip);
5047         regarray[idx] += src;
5048
5049         /* Thread. */
5050         thread_ip_inc(p);
5051 }
5052
5053 static inline void
5054 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5055 {
5056         struct thread *t = &p->threads[p->thread_id];
5057         struct instruction *ip = t->ip;
5058         uint64_t *regarray, idx, src;
5059
5060         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5061
5062         /* Structs. */
5063         regarray = instr_regarray_regarray(p, ip);
5064         idx = instr_regarray_idx_imm(p, ip);
5065         src = instr_regarray_src_hbo(t, ip);
5066         regarray[idx] += src;
5067
5068         /* Thread. */
5069         thread_ip_inc(p);
5070 }
5071
5072 static inline void
5073 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5074 {
5075         struct thread *t = &p->threads[p->thread_id];
5076         struct instruction *ip = t->ip;
5077         uint64_t *regarray, idx, src;
5078
5079         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5080
5081         /* Structs. */
5082         regarray = instr_regarray_regarray(p, ip);
5083         idx = instr_regarray_idx_imm(p, ip);
5084         src = ip->regarray.dstsrc_val;
5085         regarray[idx] += src;
5086
5087         /* Thread. */
5088         thread_ip_inc(p);
5089 }
5090
5091 /*
5092  * metarray.
5093  */
5094 static struct metarray *
5095 metarray_find(struct rte_swx_pipeline *p, const char *name);
5096
5097 static int
5098 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5099                             struct action *action,
5100                             char **tokens,
5101                             int n_tokens,
5102                             struct instruction *instr,
5103                             struct instruction_data *data __rte_unused)
5104 {
5105         char *metarray = tokens[1], *idx = tokens[2];
5106         struct metarray *m;
5107         struct field *fidx;
5108         uint32_t idx_struct_id, idx_val;
5109
5110         CHECK(n_tokens == 3, EINVAL);
5111
5112         m = metarray_find(p, metarray);
5113         CHECK(m, EINVAL);
5114
5115         /* METPREFETCH_H, METPREFETCH_M. */
5116         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5117         if (fidx) {
5118                 CHECK(!fidx->var_size, EINVAL);
5119
5120                 instr->type = INSTR_METPREFETCH_M;
5121                 if (idx[0] == 'h')
5122                         instr->type = INSTR_METPREFETCH_H;
5123
5124                 instr->meter.metarray_id = m->id;
5125                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5126                 instr->meter.idx.n_bits = fidx->n_bits;
5127                 instr->meter.idx.offset = fidx->offset / 8;
5128                 return 0;
5129         }
5130
5131         /* METPREFETCH_I. */
5132         idx_val = strtoul(idx, &idx, 0);
5133         CHECK(!idx[0], EINVAL);
5134
5135         instr->type = INSTR_METPREFETCH_I;
5136         instr->meter.metarray_id = m->id;
5137         instr->meter.idx_val = idx_val;
5138         return 0;
5139 }
5140
5141 static int
5142 instr_meter_translate(struct rte_swx_pipeline *p,
5143                       struct action *action,
5144                       char **tokens,
5145                       int n_tokens,
5146                       struct instruction *instr,
5147                       struct instruction_data *data __rte_unused)
5148 {
5149         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5150         char *color_in = tokens[4], *color_out = tokens[5];
5151         struct metarray *m;
5152         struct field *fidx, *flength, *fcin, *fcout;
5153         uint32_t idx_struct_id, length_struct_id;
5154         uint32_t color_in_struct_id, color_out_struct_id;
5155
5156         CHECK(n_tokens == 6, EINVAL);
5157
5158         m = metarray_find(p, metarray);
5159         CHECK(m, EINVAL);
5160
5161         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5162
5163         flength = struct_field_parse(p, action, length, &length_struct_id);
5164         CHECK(flength, EINVAL);
5165         CHECK(!flength->var_size, EINVAL);
5166
5167         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5168
5169         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5170         CHECK(fcout, EINVAL);
5171         CHECK(!fcout->var_size, EINVAL);
5172
5173         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5174         if (fidx && fcin) {
5175                 CHECK(!fidx->var_size, EINVAL);
5176                 CHECK(!fcin->var_size, EINVAL);
5177
5178                 instr->type = INSTR_METER_MMM;
5179                 if (idx[0] == 'h' && length[0] == 'h')
5180                         instr->type = INSTR_METER_HHM;
5181                 if (idx[0] == 'h' && length[0] != 'h')
5182                         instr->type = INSTR_METER_HMM;
5183                 if (idx[0] != 'h' && length[0] == 'h')
5184                         instr->type = INSTR_METER_MHM;
5185
5186                 instr->meter.metarray_id = m->id;
5187
5188                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5189                 instr->meter.idx.n_bits = fidx->n_bits;
5190                 instr->meter.idx.offset = fidx->offset / 8;
5191
5192                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5193                 instr->meter.length.n_bits = flength->n_bits;
5194                 instr->meter.length.offset = flength->offset / 8;
5195
5196                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5197                 instr->meter.color_in.n_bits = fcin->n_bits;
5198                 instr->meter.color_in.offset = fcin->offset / 8;
5199
5200                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5201                 instr->meter.color_out.n_bits = fcout->n_bits;
5202                 instr->meter.color_out.offset = fcout->offset / 8;
5203
5204                 return 0;
5205         }
5206
5207         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5208         if (fidx && !fcin) {
5209                 uint32_t color_in_val;
5210
5211                 CHECK(!fidx->var_size, EINVAL);
5212
5213                 color_in_val = strtoul(color_in, &color_in, 0);
5214                 CHECK(!color_in[0], EINVAL);
5215
5216                 instr->type = INSTR_METER_MMI;
5217                 if (idx[0] == 'h' && length[0] == 'h')
5218                         instr->type = INSTR_METER_HHI;
5219                 if (idx[0] == 'h' && length[0] != 'h')
5220                         instr->type = INSTR_METER_HMI;
5221                 if (idx[0] != 'h' && length[0] == 'h')
5222                         instr->type = INSTR_METER_MHI;
5223
5224                 instr->meter.metarray_id = m->id;
5225
5226                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5227                 instr->meter.idx.n_bits = fidx->n_bits;
5228                 instr->meter.idx.offset = fidx->offset / 8;
5229
5230                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5231                 instr->meter.length.n_bits = flength->n_bits;
5232                 instr->meter.length.offset = flength->offset / 8;
5233
5234                 instr->meter.color_in_val = color_in_val;
5235
5236                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5237                 instr->meter.color_out.n_bits = fcout->n_bits;
5238                 instr->meter.color_out.offset = fcout->offset / 8;
5239
5240                 return 0;
5241         }
5242
5243         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5244         if (!fidx && fcin) {
5245                 uint32_t idx_val;
5246
5247                 idx_val = strtoul(idx, &idx, 0);
5248                 CHECK(!idx[0], EINVAL);
5249
5250                 CHECK(!fcin->var_size, EINVAL);
5251
5252                 instr->type = INSTR_METER_IMM;
5253                 if (length[0] == 'h')
5254                         instr->type = INSTR_METER_IHM;
5255
5256                 instr->meter.metarray_id = m->id;
5257
5258                 instr->meter.idx_val = idx_val;
5259
5260                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5261                 instr->meter.length.n_bits = flength->n_bits;
5262                 instr->meter.length.offset = flength->offset / 8;
5263
5264                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5265                 instr->meter.color_in.n_bits = fcin->n_bits;
5266                 instr->meter.color_in.offset = fcin->offset / 8;
5267
5268                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5269                 instr->meter.color_out.n_bits = fcout->n_bits;
5270                 instr->meter.color_out.offset = fcout->offset / 8;
5271
5272                 return 0;
5273         }
5274
5275         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5276         if (!fidx && !fcin) {
5277                 uint32_t idx_val, color_in_val;
5278
5279                 idx_val = strtoul(idx, &idx, 0);
5280                 CHECK(!idx[0], EINVAL);
5281
5282                 color_in_val = strtoul(color_in, &color_in, 0);
5283                 CHECK(!color_in[0], EINVAL);
5284
5285                 instr->type = INSTR_METER_IMI;
5286                 if (length[0] == 'h')
5287                         instr->type = INSTR_METER_IHI;
5288
5289                 instr->meter.metarray_id = m->id;
5290
5291                 instr->meter.idx_val = idx_val;
5292
5293                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5294                 instr->meter.length.n_bits = flength->n_bits;
5295                 instr->meter.length.offset = flength->offset / 8;
5296
5297                 instr->meter.color_in_val = color_in_val;
5298
5299                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5300                 instr->meter.color_out.n_bits = fcout->n_bits;
5301                 instr->meter.color_out.offset = fcout->offset / 8;
5302
5303                 return 0;
5304         }
5305
5306         CHECK(0, EINVAL);
5307 }
5308
5309 static inline struct meter *
5310 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5311 {
5312         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5313
5314         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5315         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5316         uint64_t idx64 = *idx64_ptr;
5317         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5318         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5319
5320         return &r->metarray[idx];
5321 }
5322
5323 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5324
5325 static inline struct meter *
5326 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5327 {
5328         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5329
5330         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5331         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5332         uint64_t idx64 = *idx64_ptr;
5333         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5334
5335         return &r->metarray[idx];
5336 }
5337
5338 #else
5339
5340 #define instr_meter_idx_nbo instr_meter_idx_hbo
5341
5342 #endif
5343
5344 static inline struct meter *
5345 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5346 {
5347         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5348
5349         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5350
5351         return &r->metarray[idx];
5352 }
5353
5354 static inline uint32_t
5355 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5356 {
5357         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5358         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5359         uint64_t src64 = *src64_ptr;
5360         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5361         uint64_t src = src64 & src64_mask;
5362
5363         return (uint32_t)src;
5364 }
5365
5366 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5367
5368 static inline uint32_t
5369 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5370 {
5371         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5372         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5373         uint64_t src64 = *src64_ptr;
5374         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5375
5376         return (uint32_t)src;
5377 }
5378
5379 #else
5380
5381 #define instr_meter_length_nbo instr_meter_length_hbo
5382
5383 #endif
5384
5385 static inline enum rte_color
5386 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5387 {
5388         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5389         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5390         uint64_t src64 = *src64_ptr;
5391         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5392         uint64_t src = src64 & src64_mask;
5393
5394         return (enum rte_color)src;
5395 }
5396
5397 static inline void
5398 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5399 {
5400         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5401         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5402         uint64_t dst64 = *dst64_ptr;
5403         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5404
5405         uint64_t src = (uint64_t)color_out;
5406
5407         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5408 }
5409
5410 static inline void
5411 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5412 {
5413         struct thread *t = &p->threads[p->thread_id];
5414         struct instruction *ip = t->ip;
5415         struct meter *m;
5416
5417         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5418
5419         /* Structs. */
5420         m = instr_meter_idx_nbo(p, t, ip);
5421         rte_prefetch0(m);
5422
5423         /* Thread. */
5424         thread_ip_inc(p);
5425 }
5426
5427 static inline void
5428 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5429 {
5430         struct thread *t = &p->threads[p->thread_id];
5431         struct instruction *ip = t->ip;
5432         struct meter *m;
5433
5434         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5435
5436         /* Structs. */
5437         m = instr_meter_idx_hbo(p, t, ip);
5438         rte_prefetch0(m);
5439
5440         /* Thread. */
5441         thread_ip_inc(p);
5442 }
5443
5444 static inline void
5445 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5446 {
5447         struct thread *t = &p->threads[p->thread_id];
5448         struct instruction *ip = t->ip;
5449         struct meter *m;
5450
5451         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5452
5453         /* Structs. */
5454         m = instr_meter_idx_imm(p, ip);
5455         rte_prefetch0(m);
5456
5457         /* Thread. */
5458         thread_ip_inc(p);
5459 }
5460
5461 static inline void
5462 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5463 {
5464         struct thread *t = &p->threads[p->thread_id];
5465         struct instruction *ip = t->ip;
5466         struct meter *m;
5467         uint64_t time, n_pkts, n_bytes;
5468         uint32_t length;
5469         enum rte_color color_in, color_out;
5470
5471         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5472
5473         /* Structs. */
5474         m = instr_meter_idx_nbo(p, t, ip);
5475         rte_prefetch0(m->n_pkts);
5476         time = rte_get_tsc_cycles();
5477         length = instr_meter_length_nbo(t, ip);
5478         color_in = instr_meter_color_in_hbo(t, ip);
5479
5480         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5481                 &m->profile->profile,
5482                 time,
5483                 length,
5484                 color_in);
5485
5486         color_out &= m->color_mask;
5487
5488         n_pkts = m->n_pkts[color_out];
5489         n_bytes = m->n_bytes[color_out];
5490
5491         instr_meter_color_out_hbo_set(t, ip, color_out);
5492
5493         m->n_pkts[color_out] = n_pkts + 1;
5494         m->n_bytes[color_out] = n_bytes + length;
5495
5496         /* Thread. */
5497         thread_ip_inc(p);
5498 }
5499
5500 static inline void
5501 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5502 {
5503         struct thread *t = &p->threads[p->thread_id];
5504         struct instruction *ip = t->ip;
5505         struct meter *m;
5506         uint64_t time, n_pkts, n_bytes;
5507         uint32_t length;
5508         enum rte_color color_in, color_out;
5509
5510         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5511
5512         /* Structs. */
5513         m = instr_meter_idx_nbo(p, t, ip);
5514         rte_prefetch0(m->n_pkts);
5515         time = rte_get_tsc_cycles();
5516         length = instr_meter_length_nbo(t, ip);
5517         color_in = (enum rte_color)ip->meter.color_in_val;
5518
5519         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5520                 &m->profile->profile,
5521                 time,
5522                 length,
5523                 color_in);
5524
5525         color_out &= m->color_mask;
5526
5527         n_pkts = m->n_pkts[color_out];
5528         n_bytes = m->n_bytes[color_out];
5529
5530         instr_meter_color_out_hbo_set(t, ip, color_out);
5531
5532         m->n_pkts[color_out] = n_pkts + 1;
5533         m->n_bytes[color_out] = n_bytes + length;
5534
5535         /* Thread. */
5536         thread_ip_inc(p);
5537 }
5538
5539 static inline void
5540 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5541 {
5542         struct thread *t = &p->threads[p->thread_id];
5543         struct instruction *ip = t->ip;
5544         struct meter *m;
5545         uint64_t time, n_pkts, n_bytes;
5546         uint32_t length;
5547         enum rte_color color_in, color_out;
5548
5549         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5550
5551         /* Structs. */
5552         m = instr_meter_idx_nbo(p, t, ip);
5553         rte_prefetch0(m->n_pkts);
5554         time = rte_get_tsc_cycles();
5555         length = instr_meter_length_hbo(t, ip);
5556         color_in = instr_meter_color_in_hbo(t, ip);
5557
5558         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5559                 &m->profile->profile,
5560                 time,
5561                 length,
5562                 color_in);
5563
5564         color_out &= m->color_mask;
5565
5566         n_pkts = m->n_pkts[color_out];
5567         n_bytes = m->n_bytes[color_out];
5568
5569         instr_meter_color_out_hbo_set(t, ip, color_out);
5570
5571         m->n_pkts[color_out] = n_pkts + 1;
5572         m->n_bytes[color_out] = n_bytes + length;
5573
5574         /* Thread. */
5575         thread_ip_inc(p);
5576 }
5577 static inline void
5578 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5579 {
5580         struct thread *t = &p->threads[p->thread_id];
5581         struct instruction *ip = t->ip;
5582         struct meter *m;
5583         uint64_t time, n_pkts, n_bytes;
5584         uint32_t length;
5585         enum rte_color color_in, color_out;
5586
5587         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5588
5589         /* Structs. */
5590         m = instr_meter_idx_nbo(p, t, ip);
5591         rte_prefetch0(m->n_pkts);
5592         time = rte_get_tsc_cycles();
5593         length = instr_meter_length_hbo(t, ip);
5594         color_in = (enum rte_color)ip->meter.color_in_val;
5595
5596         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5597                 &m->profile->profile,
5598                 time,
5599                 length,
5600                 color_in);
5601
5602         color_out &= m->color_mask;
5603
5604         n_pkts = m->n_pkts[color_out];
5605         n_bytes = m->n_bytes[color_out];
5606
5607         instr_meter_color_out_hbo_set(t, ip, color_out);
5608
5609         m->n_pkts[color_out] = n_pkts + 1;
5610         m->n_bytes[color_out] = n_bytes + length;
5611
5612         /* Thread. */
5613         thread_ip_inc(p);
5614 }
5615
5616 static inline void
5617 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5618 {
5619         struct thread *t = &p->threads[p->thread_id];
5620         struct instruction *ip = t->ip;
5621         struct meter *m;
5622         uint64_t time, n_pkts, n_bytes;
5623         uint32_t length;
5624         enum rte_color color_in, color_out;
5625
5626         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5627
5628         /* Structs. */
5629         m = instr_meter_idx_hbo(p, t, ip);
5630         rte_prefetch0(m->n_pkts);
5631         time = rte_get_tsc_cycles();
5632         length = instr_meter_length_nbo(t, ip);
5633         color_in = instr_meter_color_in_hbo(t, ip);
5634
5635         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5636                 &m->profile->profile,
5637                 time,
5638                 length,
5639                 color_in);
5640
5641         color_out &= m->color_mask;
5642
5643         n_pkts = m->n_pkts[color_out];
5644         n_bytes = m->n_bytes[color_out];
5645
5646         instr_meter_color_out_hbo_set(t, ip, color_out);
5647
5648         m->n_pkts[color_out] = n_pkts + 1;
5649         m->n_bytes[color_out] = n_bytes + length;
5650
5651         /* Thread. */
5652         thread_ip_inc(p);
5653 }
5654
5655 static inline void
5656 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5657 {
5658         struct thread *t = &p->threads[p->thread_id];
5659         struct instruction *ip = t->ip;
5660         struct meter *m;
5661         uint64_t time, n_pkts, n_bytes;
5662         uint32_t length;
5663         enum rte_color color_in, color_out;
5664
5665         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5666
5667         /* Structs. */
5668         m = instr_meter_idx_hbo(p, t, ip);
5669         rte_prefetch0(m->n_pkts);
5670         time = rte_get_tsc_cycles();
5671         length = instr_meter_length_nbo(t, ip);
5672         color_in = (enum rte_color)ip->meter.color_in_val;
5673
5674         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5675                 &m->profile->profile,
5676                 time,
5677                 length,
5678                 color_in);
5679
5680         color_out &= m->color_mask;
5681
5682         n_pkts = m->n_pkts[color_out];
5683         n_bytes = m->n_bytes[color_out];
5684
5685         instr_meter_color_out_hbo_set(t, ip, color_out);
5686
5687         m->n_pkts[color_out] = n_pkts + 1;
5688         m->n_bytes[color_out] = n_bytes + length;
5689
5690         /* Thread. */
5691         thread_ip_inc(p);
5692 }
5693
5694 static inline void
5695 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5696 {
5697         struct thread *t = &p->threads[p->thread_id];
5698         struct instruction *ip = t->ip;
5699         struct meter *m;
5700         uint64_t time, n_pkts, n_bytes;
5701         uint32_t length;
5702         enum rte_color color_in, color_out;
5703
5704         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5705
5706         /* Structs. */
5707         m = instr_meter_idx_hbo(p, t, ip);
5708         rte_prefetch0(m->n_pkts);
5709         time = rte_get_tsc_cycles();
5710         length = instr_meter_length_hbo(t, ip);
5711         color_in = instr_meter_color_in_hbo(t, ip);
5712
5713         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5714                 &m->profile->profile,
5715                 time,
5716                 length,
5717                 color_in);
5718
5719         color_out &= m->color_mask;
5720
5721         n_pkts = m->n_pkts[color_out];
5722         n_bytes = m->n_bytes[color_out];
5723
5724         instr_meter_color_out_hbo_set(t, ip, color_out);
5725
5726         m->n_pkts[color_out] = n_pkts + 1;
5727         m->n_bytes[color_out] = n_bytes + length;
5728
5729         /* Thread. */
5730         thread_ip_inc(p);
5731 }
5732
5733 static inline void
5734 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5735 {
5736         struct thread *t = &p->threads[p->thread_id];
5737         struct instruction *ip = t->ip;
5738         struct meter *m;
5739         uint64_t time, n_pkts, n_bytes;
5740         uint32_t length;
5741         enum rte_color color_in, color_out;
5742
5743         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5744
5745         /* Structs. */
5746         m = instr_meter_idx_hbo(p, t, ip);
5747         rte_prefetch0(m->n_pkts);
5748         time = rte_get_tsc_cycles();
5749         length = instr_meter_length_hbo(t, ip);
5750         color_in = (enum rte_color)ip->meter.color_in_val;
5751
5752         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5753                 &m->profile->profile,
5754                 time,
5755                 length,
5756                 color_in);
5757
5758         color_out &= m->color_mask;
5759
5760         n_pkts = m->n_pkts[color_out];
5761         n_bytes = m->n_bytes[color_out];
5762
5763         instr_meter_color_out_hbo_set(t, ip, color_out);
5764
5765         m->n_pkts[color_out] = n_pkts + 1;
5766         m->n_bytes[color_out] = n_bytes + length;
5767
5768         /* Thread. */
5769         thread_ip_inc(p);
5770 }
5771
5772 static inline void
5773 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5774 {
5775         struct thread *t = &p->threads[p->thread_id];
5776         struct instruction *ip = t->ip;
5777         struct meter *m;
5778         uint64_t time, n_pkts, n_bytes;
5779         uint32_t length;
5780         enum rte_color color_in, color_out;
5781
5782         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5783
5784         /* Structs. */
5785         m = instr_meter_idx_imm(p, ip);
5786         rte_prefetch0(m->n_pkts);
5787         time = rte_get_tsc_cycles();
5788         length = instr_meter_length_nbo(t, ip);
5789         color_in = instr_meter_color_in_hbo(t, ip);
5790
5791         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5792                 &m->profile->profile,
5793                 time,
5794                 length,
5795                 color_in);
5796
5797         color_out &= m->color_mask;
5798
5799         n_pkts = m->n_pkts[color_out];
5800         n_bytes = m->n_bytes[color_out];
5801
5802         instr_meter_color_out_hbo_set(t, ip, color_out);
5803
5804         m->n_pkts[color_out] = n_pkts + 1;
5805         m->n_bytes[color_out] = n_bytes + length;
5806
5807         /* Thread. */
5808         thread_ip_inc(p);
5809 }
5810
5811 static inline void
5812 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5813 {
5814         struct thread *t = &p->threads[p->thread_id];
5815         struct instruction *ip = t->ip;
5816         struct meter *m;
5817         uint64_t time, n_pkts, n_bytes;
5818         uint32_t length;
5819         enum rte_color color_in, color_out;
5820
5821         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5822
5823         /* Structs. */
5824         m = instr_meter_idx_imm(p, ip);
5825         rte_prefetch0(m->n_pkts);
5826         time = rte_get_tsc_cycles();
5827         length = instr_meter_length_nbo(t, ip);
5828         color_in = (enum rte_color)ip->meter.color_in_val;
5829
5830         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5831                 &m->profile->profile,
5832                 time,
5833                 length,
5834                 color_in);
5835
5836         color_out &= m->color_mask;
5837
5838         n_pkts = m->n_pkts[color_out];
5839         n_bytes = m->n_bytes[color_out];
5840
5841         instr_meter_color_out_hbo_set(t, ip, color_out);
5842
5843         m->n_pkts[color_out] = n_pkts + 1;
5844         m->n_bytes[color_out] = n_bytes + length;
5845
5846         /* Thread. */
5847         thread_ip_inc(p);
5848 }
5849
5850 static inline void
5851 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5852 {
5853         struct thread *t = &p->threads[p->thread_id];
5854         struct instruction *ip = t->ip;
5855         struct meter *m;
5856         uint64_t time, n_pkts, n_bytes;
5857         uint32_t length;
5858         enum rte_color color_in, color_out;
5859
5860         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5861
5862         /* Structs. */
5863         m = instr_meter_idx_imm(p, ip);
5864         rte_prefetch0(m->n_pkts);
5865         time = rte_get_tsc_cycles();
5866         length = instr_meter_length_hbo(t, ip);
5867         color_in = instr_meter_color_in_hbo(t, ip);
5868
5869         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5870                 &m->profile->profile,
5871                 time,
5872                 length,
5873                 color_in);
5874
5875         color_out &= m->color_mask;
5876
5877         n_pkts = m->n_pkts[color_out];
5878         n_bytes = m->n_bytes[color_out];
5879
5880         instr_meter_color_out_hbo_set(t, ip, color_out);
5881
5882         m->n_pkts[color_out] = n_pkts + 1;
5883         m->n_bytes[color_out] = n_bytes + length;
5884
5885         /* Thread. */
5886         thread_ip_inc(p);
5887 }
5888 static inline void
5889 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5890 {
5891         struct thread *t = &p->threads[p->thread_id];
5892         struct instruction *ip = t->ip;
5893         struct meter *m;
5894         uint64_t time, n_pkts, n_bytes;
5895         uint32_t length;
5896         enum rte_color color_in, color_out;
5897
5898         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5899
5900         /* Structs. */
5901         m = instr_meter_idx_imm(p, ip);
5902         rte_prefetch0(m->n_pkts);
5903         time = rte_get_tsc_cycles();
5904         length = instr_meter_length_hbo(t, ip);
5905         color_in = (enum rte_color)ip->meter.color_in_val;
5906
5907         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5908                 &m->profile->profile,
5909                 time,
5910                 length,
5911                 color_in);
5912
5913         color_out &= m->color_mask;
5914
5915         n_pkts = m->n_pkts[color_out];
5916         n_bytes = m->n_bytes[color_out];
5917
5918         instr_meter_color_out_hbo_set(t, ip, color_out);
5919
5920         m->n_pkts[color_out] = n_pkts + 1;
5921         m->n_bytes[color_out] = n_bytes + length;
5922
5923         /* Thread. */
5924         thread_ip_inc(p);
5925 }
5926
5927 /*
5928  * jmp.
5929  */
5930 static int
5931 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5932                     struct action *action __rte_unused,
5933                     char **tokens,
5934                     int n_tokens,
5935                     struct instruction *instr,
5936                     struct instruction_data *data)
5937 {
5938         CHECK(n_tokens == 2, EINVAL);
5939
5940         strcpy(data->jmp_label, tokens[1]);
5941
5942         instr->type = INSTR_JMP;
5943         instr->jmp.ip = NULL; /* Resolved later. */
5944         return 0;
5945 }
5946
5947 static int
5948 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5949                           struct action *action __rte_unused,
5950                           char **tokens,
5951                           int n_tokens,
5952                           struct instruction *instr,
5953                           struct instruction_data *data)
5954 {
5955         struct header *h;
5956
5957         CHECK(n_tokens == 3, EINVAL);
5958
5959         strcpy(data->jmp_label, tokens[1]);
5960
5961         h = header_parse(p, tokens[2]);
5962         CHECK(h, EINVAL);
5963
5964         instr->type = INSTR_JMP_VALID;
5965         instr->jmp.ip = NULL; /* Resolved later. */
5966         instr->jmp.header_id = h->id;
5967         return 0;
5968 }
5969
5970 static int
5971 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5972                             struct action *action __rte_unused,
5973                             char **tokens,
5974                             int n_tokens,
5975                             struct instruction *instr,
5976                             struct instruction_data *data)
5977 {
5978         struct header *h;
5979
5980         CHECK(n_tokens == 3, EINVAL);
5981
5982         strcpy(data->jmp_label, tokens[1]);
5983
5984         h = header_parse(p, tokens[2]);
5985         CHECK(h, EINVAL);
5986
5987         instr->type = INSTR_JMP_INVALID;
5988         instr->jmp.ip = NULL; /* Resolved later. */
5989         instr->jmp.header_id = h->id;
5990         return 0;
5991 }
5992
5993 static int
5994 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5995                         struct action *action,
5996                         char **tokens,
5997                         int n_tokens,
5998                         struct instruction *instr,
5999                         struct instruction_data *data)
6000 {
6001         CHECK(!action, EINVAL);
6002         CHECK(n_tokens == 2, EINVAL);
6003
6004         strcpy(data->jmp_label, tokens[1]);
6005
6006         instr->type = INSTR_JMP_HIT;
6007         instr->jmp.ip = NULL; /* Resolved later. */
6008         return 0;
6009 }
6010
6011 static int
6012 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6013                          struct action *action,
6014                          char **tokens,
6015                          int n_tokens,
6016                          struct instruction *instr,
6017                          struct instruction_data *data)
6018 {
6019         CHECK(!action, EINVAL);
6020         CHECK(n_tokens == 2, EINVAL);
6021
6022         strcpy(data->jmp_label, tokens[1]);
6023
6024         instr->type = INSTR_JMP_MISS;
6025         instr->jmp.ip = NULL; /* Resolved later. */
6026         return 0;
6027 }
6028
6029 static int
6030 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6031                                struct action *action,
6032                                char **tokens,
6033                                int n_tokens,
6034                                struct instruction *instr,
6035                                struct instruction_data *data)
6036 {
6037         struct action *a;
6038
6039         CHECK(!action, EINVAL);
6040         CHECK(n_tokens == 3, EINVAL);
6041
6042         strcpy(data->jmp_label, tokens[1]);
6043
6044         a = action_find(p, tokens[2]);
6045         CHECK(a, EINVAL);
6046
6047         instr->type = INSTR_JMP_ACTION_HIT;
6048         instr->jmp.ip = NULL; /* Resolved later. */
6049         instr->jmp.action_id = a->id;
6050         return 0;
6051 }
6052
6053 static int
6054 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6055                                 struct action *action,
6056                                 char **tokens,
6057                                 int n_tokens,
6058                                 struct instruction *instr,
6059                                 struct instruction_data *data)
6060 {
6061         struct action *a;
6062
6063         CHECK(!action, EINVAL);
6064         CHECK(n_tokens == 3, EINVAL);
6065
6066         strcpy(data->jmp_label, tokens[1]);
6067
6068         a = action_find(p, tokens[2]);
6069         CHECK(a, EINVAL);
6070
6071         instr->type = INSTR_JMP_ACTION_MISS;
6072         instr->jmp.ip = NULL; /* Resolved later. */
6073         instr->jmp.action_id = a->id;
6074         return 0;
6075 }
6076
6077 static int
6078 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6079                        struct action *action,
6080                        char **tokens,
6081                        int n_tokens,
6082                        struct instruction *instr,
6083                        struct instruction_data *data)
6084 {
6085         char *a = tokens[2], *b = tokens[3];
6086         struct field *fa, *fb;
6087         uint64_t b_val;
6088         uint32_t a_struct_id, b_struct_id;
6089
6090         CHECK(n_tokens == 4, EINVAL);
6091
6092         strcpy(data->jmp_label, tokens[1]);
6093
6094         fa = struct_field_parse(p, action, a, &a_struct_id);
6095         CHECK(fa, EINVAL);
6096         CHECK(!fa->var_size, EINVAL);
6097
6098         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
6099         fb = struct_field_parse(p, action, b, &b_struct_id);
6100         if (fb) {
6101                 CHECK(!fb->var_size, EINVAL);
6102
6103                 instr->type = INSTR_JMP_EQ;
6104                 if (a[0] != 'h' && b[0] == 'h')
6105                         instr->type = INSTR_JMP_EQ_MH;
6106                 if (a[0] == 'h' && b[0] != 'h')
6107                         instr->type = INSTR_JMP_EQ_HM;
6108                 if (a[0] == 'h' && b[0] == 'h')
6109                         instr->type = INSTR_JMP_EQ_HH;
6110                 instr->jmp.ip = NULL; /* Resolved later. */
6111
6112                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6113                 instr->jmp.a.n_bits = fa->n_bits;
6114                 instr->jmp.a.offset = fa->offset / 8;
6115                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6116                 instr->jmp.b.n_bits = fb->n_bits;
6117                 instr->jmp.b.offset = fb->offset / 8;
6118                 return 0;
6119         }
6120
6121         /* JMP_EQ_I. */
6122         b_val = strtoull(b, &b, 0);
6123         CHECK(!b[0], EINVAL);
6124
6125         if (a[0] == 'h')
6126                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6127
6128         instr->type = INSTR_JMP_EQ_I;
6129         instr->jmp.ip = NULL; /* Resolved later. */
6130         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6131         instr->jmp.a.n_bits = fa->n_bits;
6132         instr->jmp.a.offset = fa->offset / 8;
6133         instr->jmp.b_val = b_val;
6134         return 0;
6135 }
6136
6137 static int
6138 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6139                         struct action *action,
6140                         char **tokens,
6141                         int n_tokens,
6142                         struct instruction *instr,
6143                         struct instruction_data *data)
6144 {
6145         char *a = tokens[2], *b = tokens[3];
6146         struct field *fa, *fb;
6147         uint64_t b_val;
6148         uint32_t a_struct_id, b_struct_id;
6149
6150         CHECK(n_tokens == 4, EINVAL);
6151
6152         strcpy(data->jmp_label, tokens[1]);
6153
6154         fa = struct_field_parse(p, action, a, &a_struct_id);
6155         CHECK(fa, EINVAL);
6156         CHECK(!fa->var_size, EINVAL);
6157
6158         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
6159         fb = struct_field_parse(p, action, b, &b_struct_id);
6160         if (fb) {
6161                 CHECK(!fb->var_size, EINVAL);
6162
6163                 instr->type = INSTR_JMP_NEQ;
6164                 if (a[0] != 'h' && b[0] == 'h')
6165                         instr->type = INSTR_JMP_NEQ_MH;
6166                 if (a[0] == 'h' && b[0] != 'h')
6167                         instr->type = INSTR_JMP_NEQ_HM;
6168                 if (a[0] == 'h' && b[0] == 'h')
6169                         instr->type = INSTR_JMP_NEQ_HH;
6170                 instr->jmp.ip = NULL; /* Resolved later. */
6171
6172                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6173                 instr->jmp.a.n_bits = fa->n_bits;
6174                 instr->jmp.a.offset = fa->offset / 8;
6175                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6176                 instr->jmp.b.n_bits = fb->n_bits;
6177                 instr->jmp.b.offset = fb->offset / 8;
6178                 return 0;
6179         }
6180
6181         /* JMP_NEQ_I. */
6182         b_val = strtoull(b, &b, 0);
6183         CHECK(!b[0], EINVAL);
6184
6185         if (a[0] == 'h')
6186                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6187
6188         instr->type = INSTR_JMP_NEQ_I;
6189         instr->jmp.ip = NULL; /* Resolved later. */
6190         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6191         instr->jmp.a.n_bits = fa->n_bits;
6192         instr->jmp.a.offset = fa->offset / 8;
6193         instr->jmp.b_val = b_val;
6194         return 0;
6195 }
6196
6197 static int
6198 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6199                        struct action *action,
6200                        char **tokens,
6201                        int n_tokens,
6202                        struct instruction *instr,
6203                        struct instruction_data *data)
6204 {
6205         char *a = tokens[2], *b = tokens[3];
6206         struct field *fa, *fb;
6207         uint64_t b_val;
6208         uint32_t a_struct_id, b_struct_id;
6209
6210         CHECK(n_tokens == 4, EINVAL);
6211
6212         strcpy(data->jmp_label, tokens[1]);
6213
6214         fa = struct_field_parse(p, action, a, &a_struct_id);
6215         CHECK(fa, EINVAL);
6216         CHECK(!fa->var_size, EINVAL);
6217
6218         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6219         fb = struct_field_parse(p, action, b, &b_struct_id);
6220         if (fb) {
6221                 CHECK(!fb->var_size, EINVAL);
6222
6223                 instr->type = INSTR_JMP_LT;
6224                 if (a[0] == 'h' && b[0] != 'h')
6225                         instr->type = INSTR_JMP_LT_HM;
6226                 if (a[0] != 'h' && b[0] == 'h')
6227                         instr->type = INSTR_JMP_LT_MH;
6228                 if (a[0] == 'h' && b[0] == 'h')
6229                         instr->type = INSTR_JMP_LT_HH;
6230                 instr->jmp.ip = NULL; /* Resolved later. */
6231
6232                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6233                 instr->jmp.a.n_bits = fa->n_bits;
6234                 instr->jmp.a.offset = fa->offset / 8;
6235                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6236                 instr->jmp.b.n_bits = fb->n_bits;
6237                 instr->jmp.b.offset = fb->offset / 8;
6238                 return 0;
6239         }
6240
6241         /* JMP_LT_MI, JMP_LT_HI. */
6242         b_val = strtoull(b, &b, 0);
6243         CHECK(!b[0], EINVAL);
6244
6245         instr->type = INSTR_JMP_LT_MI;
6246         if (a[0] == 'h')
6247                 instr->type = INSTR_JMP_LT_HI;
6248         instr->jmp.ip = NULL; /* Resolved later. */
6249
6250         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6251         instr->jmp.a.n_bits = fa->n_bits;
6252         instr->jmp.a.offset = fa->offset / 8;
6253         instr->jmp.b_val = b_val;
6254         return 0;
6255 }
6256
6257 static int
6258 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6259                        struct action *action,
6260                        char **tokens,
6261                        int n_tokens,
6262                        struct instruction *instr,
6263                        struct instruction_data *data)
6264 {
6265         char *a = tokens[2], *b = tokens[3];
6266         struct field *fa, *fb;
6267         uint64_t b_val;
6268         uint32_t a_struct_id, b_struct_id;
6269
6270         CHECK(n_tokens == 4, EINVAL);
6271
6272         strcpy(data->jmp_label, tokens[1]);
6273
6274         fa = struct_field_parse(p, action, a, &a_struct_id);
6275         CHECK(fa, EINVAL);
6276         CHECK(!fa->var_size, EINVAL);
6277
6278         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6279         fb = struct_field_parse(p, action, b, &b_struct_id);
6280         if (fb) {
6281                 CHECK(!fb->var_size, EINVAL);
6282
6283                 instr->type = INSTR_JMP_GT;
6284                 if (a[0] == 'h' && b[0] != 'h')
6285                         instr->type = INSTR_JMP_GT_HM;
6286                 if (a[0] != 'h' && b[0] == 'h')
6287                         instr->type = INSTR_JMP_GT_MH;
6288                 if (a[0] == 'h' && b[0] == 'h')
6289                         instr->type = INSTR_JMP_GT_HH;
6290                 instr->jmp.ip = NULL; /* Resolved later. */
6291
6292                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6293                 instr->jmp.a.n_bits = fa->n_bits;
6294                 instr->jmp.a.offset = fa->offset / 8;
6295                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6296                 instr->jmp.b.n_bits = fb->n_bits;
6297                 instr->jmp.b.offset = fb->offset / 8;
6298                 return 0;
6299         }
6300
6301         /* JMP_GT_MI, JMP_GT_HI. */
6302         b_val = strtoull(b, &b, 0);
6303         CHECK(!b[0], EINVAL);
6304
6305         instr->type = INSTR_JMP_GT_MI;
6306         if (a[0] == 'h')
6307                 instr->type = INSTR_JMP_GT_HI;
6308         instr->jmp.ip = NULL; /* Resolved later. */
6309
6310         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6311         instr->jmp.a.n_bits = fa->n_bits;
6312         instr->jmp.a.offset = fa->offset / 8;
6313         instr->jmp.b_val = b_val;
6314         return 0;
6315 }
6316
6317 static inline void
6318 instr_jmp_exec(struct rte_swx_pipeline *p)
6319 {
6320         struct thread *t = &p->threads[p->thread_id];
6321         struct instruction *ip = t->ip;
6322
6323         TRACE("[Thread %2u] jmp\n", p->thread_id);
6324
6325         thread_ip_set(t, ip->jmp.ip);
6326 }
6327
6328 static inline void
6329 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6330 {
6331         struct thread *t = &p->threads[p->thread_id];
6332         struct instruction *ip = t->ip;
6333         uint32_t header_id = ip->jmp.header_id;
6334
6335         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6336
6337         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6338 }
6339
6340 static inline void
6341 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6342 {
6343         struct thread *t = &p->threads[p->thread_id];
6344         struct instruction *ip = t->ip;
6345         uint32_t header_id = ip->jmp.header_id;
6346
6347         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6348
6349         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6350 }
6351
6352 static inline void
6353 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6354 {
6355         struct thread *t = &p->threads[p->thread_id];
6356         struct instruction *ip = t->ip;
6357         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6358
6359         TRACE("[Thread %2u] jmph\n", p->thread_id);
6360
6361         t->ip = ip_next[t->hit];
6362 }
6363
6364 static inline void
6365 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6366 {
6367         struct thread *t = &p->threads[p->thread_id];
6368         struct instruction *ip = t->ip;
6369         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6370
6371         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6372
6373         t->ip = ip_next[t->hit];
6374 }
6375
6376 static inline void
6377 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6378 {
6379         struct thread *t = &p->threads[p->thread_id];
6380         struct instruction *ip = t->ip;
6381
6382         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6383
6384         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6385 }
6386
6387 static inline void
6388 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6389 {
6390         struct thread *t = &p->threads[p->thread_id];
6391         struct instruction *ip = t->ip;
6392
6393         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6394
6395         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6396 }
6397
6398 static inline void
6399 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6400 {
6401         struct thread *t = &p->threads[p->thread_id];
6402         struct instruction *ip = t->ip;
6403
6404         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6405
6406         JMP_CMP(t, ip, ==);
6407 }
6408
6409 static inline void
6410 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6411 {
6412         struct thread *t = &p->threads[p->thread_id];
6413         struct instruction *ip = t->ip;
6414
6415         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6416
6417         JMP_CMP_MH(t, ip, ==);
6418 }
6419
6420 static inline void
6421 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6422 {
6423         struct thread *t = &p->threads[p->thread_id];
6424         struct instruction *ip = t->ip;
6425
6426         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6427
6428         JMP_CMP_HM(t, ip, ==);
6429 }
6430
6431 static inline void
6432 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6433 {
6434         struct thread *t = &p->threads[p->thread_id];
6435         struct instruction *ip = t->ip;
6436
6437         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6438
6439         JMP_CMP_HH_FAST(t, ip, ==);
6440 }
6441
6442 static inline void
6443 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6444 {
6445         struct thread *t = &p->threads[p->thread_id];
6446         struct instruction *ip = t->ip;
6447
6448         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6449
6450         JMP_CMP_I(t, ip, ==);
6451 }
6452
6453 static inline void
6454 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6455 {
6456         struct thread *t = &p->threads[p->thread_id];
6457         struct instruction *ip = t->ip;
6458
6459         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6460
6461         JMP_CMP(t, ip, !=);
6462 }
6463
6464 static inline void
6465 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6466 {
6467         struct thread *t = &p->threads[p->thread_id];
6468         struct instruction *ip = t->ip;
6469
6470         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6471
6472         JMP_CMP_MH(t, ip, !=);
6473 }
6474
6475 static inline void
6476 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6477 {
6478         struct thread *t = &p->threads[p->thread_id];
6479         struct instruction *ip = t->ip;
6480
6481         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6482
6483         JMP_CMP_HM(t, ip, !=);
6484 }
6485
6486 static inline void
6487 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6488 {
6489         struct thread *t = &p->threads[p->thread_id];
6490         struct instruction *ip = t->ip;
6491
6492         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6493
6494         JMP_CMP_HH_FAST(t, ip, !=);
6495 }
6496
6497 static inline void
6498 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6499 {
6500         struct thread *t = &p->threads[p->thread_id];
6501         struct instruction *ip = t->ip;
6502
6503         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6504
6505         JMP_CMP_I(t, ip, !=);
6506 }
6507
6508 static inline void
6509 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6510 {
6511         struct thread *t = &p->threads[p->thread_id];
6512         struct instruction *ip = t->ip;
6513
6514         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6515
6516         JMP_CMP(t, ip, <);
6517 }
6518
6519 static inline void
6520 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6521 {
6522         struct thread *t = &p->threads[p->thread_id];
6523         struct instruction *ip = t->ip;
6524
6525         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6526
6527         JMP_CMP_MH(t, ip, <);
6528 }
6529
6530 static inline void
6531 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6532 {
6533         struct thread *t = &p->threads[p->thread_id];
6534         struct instruction *ip = t->ip;
6535
6536         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6537
6538         JMP_CMP_HM(t, ip, <);
6539 }
6540
6541 static inline void
6542 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6543 {
6544         struct thread *t = &p->threads[p->thread_id];
6545         struct instruction *ip = t->ip;
6546
6547         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6548
6549         JMP_CMP_HH(t, ip, <);
6550 }
6551
6552 static inline void
6553 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6554 {
6555         struct thread *t = &p->threads[p->thread_id];
6556         struct instruction *ip = t->ip;
6557
6558         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6559
6560         JMP_CMP_MI(t, ip, <);
6561 }
6562
6563 static inline void
6564 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6565 {
6566         struct thread *t = &p->threads[p->thread_id];
6567         struct instruction *ip = t->ip;
6568
6569         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6570
6571         JMP_CMP_HI(t, ip, <);
6572 }
6573
6574 static inline void
6575 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6576 {
6577         struct thread *t = &p->threads[p->thread_id];
6578         struct instruction *ip = t->ip;
6579
6580         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6581
6582         JMP_CMP(t, ip, >);
6583 }
6584
6585 static inline void
6586 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6587 {
6588         struct thread *t = &p->threads[p->thread_id];
6589         struct instruction *ip = t->ip;
6590
6591         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6592
6593         JMP_CMP_MH(t, ip, >);
6594 }
6595
6596 static inline void
6597 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6598 {
6599         struct thread *t = &p->threads[p->thread_id];
6600         struct instruction *ip = t->ip;
6601
6602         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6603
6604         JMP_CMP_HM(t, ip, >);
6605 }
6606
6607 static inline void
6608 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6609 {
6610         struct thread *t = &p->threads[p->thread_id];
6611         struct instruction *ip = t->ip;
6612
6613         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6614
6615         JMP_CMP_HH(t, ip, >);
6616 }
6617
6618 static inline void
6619 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6620 {
6621         struct thread *t = &p->threads[p->thread_id];
6622         struct instruction *ip = t->ip;
6623
6624         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6625
6626         JMP_CMP_MI(t, ip, >);
6627 }
6628
6629 static inline void
6630 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6631 {
6632         struct thread *t = &p->threads[p->thread_id];
6633         struct instruction *ip = t->ip;
6634
6635         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6636
6637         JMP_CMP_HI(t, ip, >);
6638 }
6639
6640 /*
6641  * return.
6642  */
6643 static int
6644 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6645                        struct action *action,
6646                        char **tokens __rte_unused,
6647                        int n_tokens,
6648                        struct instruction *instr,
6649                        struct instruction_data *data __rte_unused)
6650 {
6651         CHECK(action, EINVAL);
6652         CHECK(n_tokens == 1, EINVAL);
6653
6654         instr->type = INSTR_RETURN;
6655         return 0;
6656 }
6657
6658 static inline void
6659 instr_return_exec(struct rte_swx_pipeline *p)
6660 {
6661         struct thread *t = &p->threads[p->thread_id];
6662
6663         TRACE("[Thread %2u] return\n", p->thread_id);
6664
6665         t->ip = t->ret;
6666 }
6667
6668 static int
6669 instr_translate(struct rte_swx_pipeline *p,
6670                 struct action *action,
6671                 char *string,
6672                 struct instruction *instr,
6673                 struct instruction_data *data)
6674 {
6675         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6676         int n_tokens = 0, tpos = 0;
6677
6678         /* Parse the instruction string into tokens. */
6679         for ( ; ; ) {
6680                 char *token;
6681
6682                 token = strtok_r(string, " \t\v", &string);
6683                 if (!token)
6684                         break;
6685
6686                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6687                 CHECK_NAME(token, EINVAL);
6688
6689                 tokens[n_tokens] = token;
6690                 n_tokens++;
6691         }
6692
6693         CHECK(n_tokens, EINVAL);
6694
6695         /* Handle the optional instruction label. */
6696         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6697                 strcpy(data->label, tokens[0]);
6698
6699                 tpos += 2;
6700                 CHECK(n_tokens - tpos, EINVAL);
6701         }
6702
6703         /* Identify the instruction type. */
6704         if (!strcmp(tokens[tpos], "rx"))
6705                 return instr_rx_translate(p,
6706                                           action,
6707                                           &tokens[tpos],
6708                                           n_tokens - tpos,
6709                                           instr,
6710                                           data);
6711
6712         if (!strcmp(tokens[tpos], "tx"))
6713                 return instr_tx_translate(p,
6714                                           action,
6715                                           &tokens[tpos],
6716                                           n_tokens - tpos,
6717                                           instr,
6718                                           data);
6719
6720         if (!strcmp(tokens[tpos], "drop"))
6721                 return instr_drop_translate(p,
6722                                             action,
6723                                             &tokens[tpos],
6724                                             n_tokens - tpos,
6725                                             instr,
6726                                             data);
6727
6728         if (!strcmp(tokens[tpos], "extract"))
6729                 return instr_hdr_extract_translate(p,
6730                                                    action,
6731                                                    &tokens[tpos],
6732                                                    n_tokens - tpos,
6733                                                    instr,
6734                                                    data);
6735
6736         if (!strcmp(tokens[tpos], "lookahead"))
6737                 return instr_hdr_lookahead_translate(p,
6738                                                      action,
6739                                                      &tokens[tpos],
6740                                                      n_tokens - tpos,
6741                                                      instr,
6742                                                      data);
6743
6744         if (!strcmp(tokens[tpos], "emit"))
6745                 return instr_hdr_emit_translate(p,
6746                                                 action,
6747                                                 &tokens[tpos],
6748                                                 n_tokens - tpos,
6749                                                 instr,
6750                                                 data);
6751
6752         if (!strcmp(tokens[tpos], "validate"))
6753                 return instr_hdr_validate_translate(p,
6754                                                     action,
6755                                                     &tokens[tpos],
6756                                                     n_tokens - tpos,
6757                                                     instr,
6758                                                     data);
6759
6760         if (!strcmp(tokens[tpos], "invalidate"))
6761                 return instr_hdr_invalidate_translate(p,
6762                                                       action,
6763                                                       &tokens[tpos],
6764                                                       n_tokens - tpos,
6765                                                       instr,
6766                                                       data);
6767
6768         if (!strcmp(tokens[tpos], "mov"))
6769                 return instr_mov_translate(p,
6770                                            action,
6771                                            &tokens[tpos],
6772                                            n_tokens - tpos,
6773                                            instr,
6774                                            data);
6775
6776         if (!strcmp(tokens[tpos], "add"))
6777                 return instr_alu_add_translate(p,
6778                                                action,
6779                                                &tokens[tpos],
6780                                                n_tokens - tpos,
6781                                                instr,
6782                                                data);
6783
6784         if (!strcmp(tokens[tpos], "sub"))
6785                 return instr_alu_sub_translate(p,
6786                                                action,
6787                                                &tokens[tpos],
6788                                                n_tokens - tpos,
6789                                                instr,
6790                                                data);
6791
6792         if (!strcmp(tokens[tpos], "ckadd"))
6793                 return instr_alu_ckadd_translate(p,
6794                                                  action,
6795                                                  &tokens[tpos],
6796                                                  n_tokens - tpos,
6797                                                  instr,
6798                                                  data);
6799
6800         if (!strcmp(tokens[tpos], "cksub"))
6801                 return instr_alu_cksub_translate(p,
6802                                                  action,
6803                                                  &tokens[tpos],
6804                                                  n_tokens - tpos,
6805                                                  instr,
6806                                                  data);
6807
6808         if (!strcmp(tokens[tpos], "and"))
6809                 return instr_alu_and_translate(p,
6810                                                action,
6811                                                &tokens[tpos],
6812                                                n_tokens - tpos,
6813                                                instr,
6814                                                data);
6815
6816         if (!strcmp(tokens[tpos], "or"))
6817                 return instr_alu_or_translate(p,
6818                                               action,
6819                                               &tokens[tpos],
6820                                               n_tokens - tpos,
6821                                               instr,
6822                                               data);
6823
6824         if (!strcmp(tokens[tpos], "xor"))
6825                 return instr_alu_xor_translate(p,
6826                                                action,
6827                                                &tokens[tpos],
6828                                                n_tokens - tpos,
6829                                                instr,
6830                                                data);
6831
6832         if (!strcmp(tokens[tpos], "shl"))
6833                 return instr_alu_shl_translate(p,
6834                                                action,
6835                                                &tokens[tpos],
6836                                                n_tokens - tpos,
6837                                                instr,
6838                                                data);
6839
6840         if (!strcmp(tokens[tpos], "shr"))
6841                 return instr_alu_shr_translate(p,
6842                                                action,
6843                                                &tokens[tpos],
6844                                                n_tokens - tpos,
6845                                                instr,
6846                                                data);
6847
6848         if (!strcmp(tokens[tpos], "regprefetch"))
6849                 return instr_regprefetch_translate(p,
6850                                                    action,
6851                                                    &tokens[tpos],
6852                                                    n_tokens - tpos,
6853                                                    instr,
6854                                                    data);
6855
6856         if (!strcmp(tokens[tpos], "regrd"))
6857                 return instr_regrd_translate(p,
6858                                              action,
6859                                              &tokens[tpos],
6860                                              n_tokens - tpos,
6861                                              instr,
6862                                              data);
6863
6864         if (!strcmp(tokens[tpos], "regwr"))
6865                 return instr_regwr_translate(p,
6866                                              action,
6867                                              &tokens[tpos],
6868                                              n_tokens - tpos,
6869                                              instr,
6870                                              data);
6871
6872         if (!strcmp(tokens[tpos], "regadd"))
6873                 return instr_regadd_translate(p,
6874                                               action,
6875                                               &tokens[tpos],
6876                                               n_tokens - tpos,
6877                                               instr,
6878                                               data);
6879
6880         if (!strcmp(tokens[tpos], "metprefetch"))
6881                 return instr_metprefetch_translate(p,
6882                                                    action,
6883                                                    &tokens[tpos],
6884                                                    n_tokens - tpos,
6885                                                    instr,
6886                                                    data);
6887
6888         if (!strcmp(tokens[tpos], "meter"))
6889                 return instr_meter_translate(p,
6890                                              action,
6891                                              &tokens[tpos],
6892                                              n_tokens - tpos,
6893                                              instr,
6894                                              data);
6895
6896         if (!strcmp(tokens[tpos], "table"))
6897                 return instr_table_translate(p,
6898                                              action,
6899                                              &tokens[tpos],
6900                                              n_tokens - tpos,
6901                                              instr,
6902                                              data);
6903
6904         if (!strcmp(tokens[tpos], "learn"))
6905                 return instr_learn_translate(p,
6906                                              action,
6907                                              &tokens[tpos],
6908                                              n_tokens - tpos,
6909                                              instr,
6910                                              data);
6911
6912         if (!strcmp(tokens[tpos], "forget"))
6913                 return instr_forget_translate(p,
6914                                               action,
6915                                               &tokens[tpos],
6916                                               n_tokens - tpos,
6917                                               instr,
6918                                               data);
6919
6920         if (!strcmp(tokens[tpos], "extern"))
6921                 return instr_extern_translate(p,
6922                                               action,
6923                                               &tokens[tpos],
6924                                               n_tokens - tpos,
6925                                               instr,
6926                                               data);
6927
6928         if (!strcmp(tokens[tpos], "jmp"))
6929                 return instr_jmp_translate(p,
6930                                            action,
6931                                            &tokens[tpos],
6932                                            n_tokens - tpos,
6933                                            instr,
6934                                            data);
6935
6936         if (!strcmp(tokens[tpos], "jmpv"))
6937                 return instr_jmp_valid_translate(p,
6938                                                  action,
6939                                                  &tokens[tpos],
6940                                                  n_tokens - tpos,
6941                                                  instr,
6942                                                  data);
6943
6944         if (!strcmp(tokens[tpos], "jmpnv"))
6945                 return instr_jmp_invalid_translate(p,
6946                                                    action,
6947                                                    &tokens[tpos],
6948                                                    n_tokens - tpos,
6949                                                    instr,
6950                                                    data);
6951
6952         if (!strcmp(tokens[tpos], "jmph"))
6953                 return instr_jmp_hit_translate(p,
6954                                                action,
6955                                                &tokens[tpos],
6956                                                n_tokens - tpos,
6957                                                instr,
6958                                                data);
6959
6960         if (!strcmp(tokens[tpos], "jmpnh"))
6961                 return instr_jmp_miss_translate(p,
6962                                                 action,
6963                                                 &tokens[tpos],
6964                                                 n_tokens - tpos,
6965                                                 instr,
6966                                                 data);
6967
6968         if (!strcmp(tokens[tpos], "jmpa"))
6969                 return instr_jmp_action_hit_translate(p,
6970                                                       action,
6971                                                       &tokens[tpos],
6972                                                       n_tokens - tpos,
6973                                                       instr,
6974                                                       data);
6975
6976         if (!strcmp(tokens[tpos], "jmpna"))
6977                 return instr_jmp_action_miss_translate(p,
6978                                                        action,
6979                                                        &tokens[tpos],
6980                                                        n_tokens - tpos,
6981                                                        instr,
6982                                                        data);
6983
6984         if (!strcmp(tokens[tpos], "jmpeq"))
6985                 return instr_jmp_eq_translate(p,
6986                                               action,
6987                                               &tokens[tpos],
6988                                               n_tokens - tpos,
6989                                               instr,
6990                                               data);
6991
6992         if (!strcmp(tokens[tpos], "jmpneq"))
6993                 return instr_jmp_neq_translate(p,
6994                                                action,
6995                                                &tokens[tpos],
6996                                                n_tokens - tpos,
6997                                                instr,
6998                                                data);
6999
7000         if (!strcmp(tokens[tpos], "jmplt"))
7001                 return instr_jmp_lt_translate(p,
7002                                               action,
7003                                               &tokens[tpos],
7004                                               n_tokens - tpos,
7005                                               instr,
7006                                               data);
7007
7008         if (!strcmp(tokens[tpos], "jmpgt"))
7009                 return instr_jmp_gt_translate(p,
7010                                               action,
7011                                               &tokens[tpos],
7012                                               n_tokens - tpos,
7013                                               instr,
7014                                               data);
7015
7016         if (!strcmp(tokens[tpos], "return"))
7017                 return instr_return_translate(p,
7018                                               action,
7019                                               &tokens[tpos],
7020                                               n_tokens - tpos,
7021                                               instr,
7022                                               data);
7023
7024         CHECK(0, EINVAL);
7025 }
7026
7027 static struct instruction_data *
7028 label_find(struct instruction_data *data, uint32_t n, const char *label)
7029 {
7030         uint32_t i;
7031
7032         for (i = 0; i < n; i++)
7033                 if (!strcmp(label, data[i].label))
7034                         return &data[i];
7035
7036         return NULL;
7037 }
7038
7039 static uint32_t
7040 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7041 {
7042         uint32_t count = 0, i;
7043
7044         if (!label[0])
7045                 return 0;
7046
7047         for (i = 0; i < n; i++)
7048                 if (!strcmp(label, data[i].jmp_label))
7049                         count++;
7050
7051         return count;
7052 }
7053
7054 static int
7055 instr_label_check(struct instruction_data *instruction_data,
7056                   uint32_t n_instructions)
7057 {
7058         uint32_t i;
7059
7060         /* Check that all instruction labels are unique. */
7061         for (i = 0; i < n_instructions; i++) {
7062                 struct instruction_data *data = &instruction_data[i];
7063                 char *label = data->label;
7064                 uint32_t j;
7065
7066                 if (!label[0])
7067                         continue;
7068
7069                 for (j = i + 1; j < n_instructions; j++)
7070                         CHECK(strcmp(label, data[j].label), EINVAL);
7071         }
7072
7073         /* Get users for each instruction label. */
7074         for (i = 0; i < n_instructions; i++) {
7075                 struct instruction_data *data = &instruction_data[i];
7076                 char *label = data->label;
7077
7078                 data->n_users = label_is_used(instruction_data,
7079                                               n_instructions,
7080                                               label);
7081         }
7082
7083         return 0;
7084 }
7085
7086 static int
7087 instr_jmp_resolve(struct instruction *instructions,
7088                   struct instruction_data *instruction_data,
7089                   uint32_t n_instructions)
7090 {
7091         uint32_t i;
7092
7093         for (i = 0; i < n_instructions; i++) {
7094                 struct instruction *instr = &instructions[i];
7095                 struct instruction_data *data = &instruction_data[i];
7096                 struct instruction_data *found;
7097
7098                 if (!instruction_is_jmp(instr))
7099                         continue;
7100
7101                 found = label_find(instruction_data,
7102                                    n_instructions,
7103                                    data->jmp_label);
7104                 CHECK(found, EINVAL);
7105
7106                 instr->jmp.ip = &instructions[found - instruction_data];
7107         }
7108
7109         return 0;
7110 }
7111
7112 static int
7113 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7114              struct action *a,
7115              struct instruction *instr,
7116              struct instruction_data *data __rte_unused,
7117              uint32_t n_instructions)
7118 {
7119         if (!a) {
7120                 enum instruction_type type;
7121                 uint32_t i;
7122
7123                 /* Check that the first instruction is rx. */
7124                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7125
7126                 /* Check that there is at least one tx instruction. */
7127                 for (i = 0; i < n_instructions; i++) {
7128                         type = instr[i].type;
7129
7130                         if (instruction_is_tx(type))
7131                                 break;
7132                 }
7133                 CHECK(i < n_instructions, EINVAL);
7134
7135                 /* Check that the last instruction is either tx or unconditional
7136                  * jump.
7137                  */
7138                 type = instr[n_instructions - 1].type;
7139                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
7140         }
7141
7142         if (a) {
7143                 enum instruction_type type;
7144                 uint32_t i;
7145
7146                 /* Check that there is at least one return or tx instruction. */
7147                 for (i = 0; i < n_instructions; i++) {
7148                         type = instr[i].type;
7149
7150                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
7151                                 break;
7152                 }
7153                 CHECK(i < n_instructions, EINVAL);
7154         }
7155
7156         return 0;
7157 }
7158
7159 static uint32_t
7160 instr_compact(struct instruction *instructions,
7161               struct instruction_data *instruction_data,
7162               uint32_t n_instructions)
7163 {
7164         uint32_t i, pos = 0;
7165
7166         /* Eliminate the invalid instructions that have been optimized out. */
7167         for (i = 0; i < n_instructions; i++) {
7168                 struct instruction *instr = &instructions[i];
7169                 struct instruction_data *data = &instruction_data[i];
7170
7171                 if (data->invalid)
7172                         continue;
7173
7174                 if (i != pos) {
7175                         memcpy(&instructions[pos], instr, sizeof(*instr));
7176                         memcpy(&instruction_data[pos], data, sizeof(*data));
7177                 }
7178
7179                 pos++;
7180         }
7181
7182         return pos;
7183 }
7184
7185 static int
7186 instr_pattern_extract_many_search(struct instruction *instr,
7187                                   struct instruction_data *data,
7188                                   uint32_t n_instr,
7189                                   uint32_t *n_pattern_instr)
7190 {
7191         uint32_t i;
7192
7193         for (i = 0; i < n_instr; i++) {
7194                 if (data[i].invalid)
7195                         break;
7196
7197                 if (instr[i].type != INSTR_HDR_EXTRACT)
7198                         break;
7199
7200                 if (i == RTE_DIM(instr->io.hdr.header_id))
7201                         break;
7202
7203                 if (i && data[i].n_users)
7204                         break;
7205         }
7206
7207         if (i < 2)
7208                 return 0;
7209
7210         *n_pattern_instr = i;
7211         return 1;
7212 }
7213
7214 static void
7215 instr_pattern_extract_many_replace(struct instruction *instr,
7216                                    struct instruction_data *data,
7217                                    uint32_t n_instr)
7218 {
7219         uint32_t i;
7220
7221         for (i = 1; i < n_instr; i++) {
7222                 instr[0].type++;
7223                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7224                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7225                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7226
7227                 data[i].invalid = 1;
7228         }
7229 }
7230
7231 static uint32_t
7232 instr_pattern_extract_many_optimize(struct instruction *instructions,
7233                                     struct instruction_data *instruction_data,
7234                                     uint32_t n_instructions)
7235 {
7236         uint32_t i;
7237
7238         for (i = 0; i < n_instructions; ) {
7239                 struct instruction *instr = &instructions[i];
7240                 struct instruction_data *data = &instruction_data[i];
7241                 uint32_t n_instr = 0;
7242                 int detected;
7243
7244                 /* Extract many. */
7245                 detected = instr_pattern_extract_many_search(instr,
7246                                                              data,
7247                                                              n_instructions - i,
7248                                                              &n_instr);
7249                 if (detected) {
7250                         instr_pattern_extract_many_replace(instr,
7251                                                            data,
7252                                                            n_instr);
7253                         i += n_instr;
7254                         continue;
7255                 }
7256
7257                 /* No pattern starting at the current instruction. */
7258                 i++;
7259         }
7260
7261         /* Eliminate the invalid instructions that have been optimized out. */
7262         n_instructions = instr_compact(instructions,
7263                                        instruction_data,
7264                                        n_instructions);
7265
7266         return n_instructions;
7267 }
7268
7269 static int
7270 instr_pattern_emit_many_tx_search(struct instruction *instr,
7271                                   struct instruction_data *data,
7272                                   uint32_t n_instr,
7273                                   uint32_t *n_pattern_instr)
7274 {
7275         uint32_t i;
7276
7277         for (i = 0; i < n_instr; i++) {
7278                 if (data[i].invalid)
7279                         break;
7280
7281                 if (instr[i].type != INSTR_HDR_EMIT)
7282                         break;
7283
7284                 if (i == RTE_DIM(instr->io.hdr.header_id))
7285                         break;
7286
7287                 if (i && data[i].n_users)
7288                         break;
7289         }
7290
7291         if (!i)
7292                 return 0;
7293
7294         if (!instruction_is_tx(instr[i].type))
7295                 return 0;
7296
7297         if (data[i].n_users)
7298                 return 0;
7299
7300         i++;
7301
7302         *n_pattern_instr = i;
7303         return 1;
7304 }
7305
7306 static void
7307 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7308                                    struct instruction_data *data,
7309                                    uint32_t n_instr)
7310 {
7311         uint32_t i;
7312
7313         /* Any emit instruction in addition to the first one. */
7314         for (i = 1; i < n_instr - 1; i++) {
7315                 instr[0].type++;
7316                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7317                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7318                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7319
7320                 data[i].invalid = 1;
7321         }
7322
7323         /* The TX instruction is the last one in the pattern. */
7324         instr[0].type++;
7325         instr[0].io.io.offset = instr[i].io.io.offset;
7326         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7327         data[i].invalid = 1;
7328 }
7329
7330 static uint32_t
7331 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7332                                     struct instruction_data *instruction_data,
7333                                     uint32_t n_instructions)
7334 {
7335         uint32_t i;
7336
7337         for (i = 0; i < n_instructions; ) {
7338                 struct instruction *instr = &instructions[i];
7339                 struct instruction_data *data = &instruction_data[i];
7340                 uint32_t n_instr = 0;
7341                 int detected;
7342
7343                 /* Emit many + TX. */
7344                 detected = instr_pattern_emit_many_tx_search(instr,
7345                                                              data,
7346                                                              n_instructions - i,
7347                                                              &n_instr);
7348                 if (detected) {
7349                         instr_pattern_emit_many_tx_replace(instr,
7350                                                            data,
7351                                                            n_instr);
7352                         i += n_instr;
7353                         continue;
7354                 }
7355
7356                 /* No pattern starting at the current instruction. */
7357                 i++;
7358         }
7359
7360         /* Eliminate the invalid instructions that have been optimized out. */
7361         n_instructions = instr_compact(instructions,
7362                                        instruction_data,
7363                                        n_instructions);
7364
7365         return n_instructions;
7366 }
7367
7368 static uint32_t
7369 action_arg_src_mov_count(struct action *a,
7370                          uint32_t arg_id,
7371                          struct instruction *instructions,
7372                          struct instruction_data *instruction_data,
7373                          uint32_t n_instructions);
7374
7375 static int
7376 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7377                                       struct action *a,
7378                                       struct instruction *instr,
7379                                       struct instruction_data *data,
7380                                       uint32_t n_instr,
7381                                       struct instruction *instructions,
7382                                       struct instruction_data *instruction_data,
7383                                       uint32_t n_instructions,
7384                                       uint32_t *n_pattern_instr)
7385 {
7386         struct header *h;
7387         uint32_t src_field_id, i, j;
7388
7389         /* Prerequisites. */
7390         if (!a || !a->st)
7391                 return 0;
7392
7393         /* First instruction: MOV_HM. */
7394         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7395                 return 0;
7396
7397         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7398         if (!h || h->st->var_size)
7399                 return 0;
7400
7401         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7402                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7403                         break;
7404
7405         if (src_field_id == a->st->n_fields)
7406                 return 0;
7407
7408         if (instr[0].mov.dst.offset ||
7409             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7410             instr[0].mov.src.struct_id ||
7411             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7412             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7413                 return 0;
7414
7415         if ((n_instr < h->st->n_fields + 1) ||
7416              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7417                 return 0;
7418
7419         /* Subsequent instructions: MOV_HM. */
7420         for (i = 1; i < h->st->n_fields; i++)
7421                 if (data[i].invalid ||
7422                     data[i].n_users ||
7423                     (instr[i].type != INSTR_MOV_HM) ||
7424                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7425                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7426                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7427                     instr[i].mov.src.struct_id ||
7428                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7429                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7430                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7431                         return 0;
7432
7433         /* Last instruction: HDR_VALIDATE. */
7434         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7435             (instr[i].valid.header_id != h->id))
7436                 return 0;
7437
7438         /* Check that none of the action args that are used as source for this
7439          * DMA transfer are not used as source in any other mov instruction.
7440          */
7441         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7442                 uint32_t n_users;
7443
7444                 n_users = action_arg_src_mov_count(a,
7445                                                    j,
7446                                                    instructions,
7447                                                    instruction_data,
7448                                                    n_instructions);
7449                 if (n_users > 1)
7450                         return 0;
7451         }
7452
7453         *n_pattern_instr = 1 + i;
7454         return 1;
7455 }
7456
7457 static void
7458 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7459                                        struct action *a,
7460                                        struct instruction *instr,
7461                                        struct instruction_data *data,
7462                                        uint32_t n_instr)
7463 {
7464         struct header *h;
7465         uint32_t src_field_id, src_offset, i;
7466
7467         /* Read from the instructions before they are modified. */
7468         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7469         if (!h)
7470                 return;
7471
7472         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7473                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7474                         break;
7475
7476         if (src_field_id == a->st->n_fields)
7477                 return;
7478
7479         src_offset = instr[0].mov.src.offset;
7480
7481         /* Modify the instructions. */
7482         instr[0].type = INSTR_DMA_HT;
7483         instr[0].dma.dst.header_id[0] = h->id;
7484         instr[0].dma.dst.struct_id[0] = h->struct_id;
7485         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7486         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7487
7488         for (i = 1; i < n_instr; i++)
7489                 data[i].invalid = 1;
7490
7491         /* Update the endianness of the action arguments to header endianness. */
7492         for (i = 0; i < h->st->n_fields; i++)
7493                 a->args_endianness[src_field_id + i] = 1;
7494 }
7495
7496 static uint32_t
7497 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7498                                         struct action *a,
7499                                         struct instruction *instructions,
7500                                         struct instruction_data *instruction_data,
7501                                         uint32_t n_instructions)
7502 {
7503         uint32_t i;
7504
7505         if (!a || !a->st)
7506                 return n_instructions;
7507
7508         for (i = 0; i < n_instructions; ) {
7509                 struct instruction *instr = &instructions[i];
7510                 struct instruction_data *data = &instruction_data[i];
7511                 uint32_t n_instr = 0;
7512                 int detected;
7513
7514                 /* Mov all + validate. */
7515                 detected = instr_pattern_mov_all_validate_search(p,
7516                                                                  a,
7517                                                                  instr,
7518                                                                  data,
7519                                                                  n_instructions - i,
7520                                                                  instructions,
7521                                                                  instruction_data,
7522                                                                  n_instructions,
7523                                                                  &n_instr);
7524                 if (detected) {
7525                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7526                         i += n_instr;
7527                         continue;
7528                 }
7529
7530                 /* No pattern starting at the current instruction. */
7531                 i++;
7532         }
7533
7534         /* Eliminate the invalid instructions that have been optimized out. */
7535         n_instructions = instr_compact(instructions,
7536                                        instruction_data,
7537                                        n_instructions);
7538
7539         return n_instructions;
7540 }
7541
7542 static int
7543 instr_pattern_dma_many_search(struct instruction *instr,
7544                               struct instruction_data *data,
7545                               uint32_t n_instr,
7546                               uint32_t *n_pattern_instr)
7547 {
7548         uint32_t i;
7549
7550         for (i = 0; i < n_instr; i++) {
7551                 if (data[i].invalid)
7552                         break;
7553
7554                 if (instr[i].type != INSTR_DMA_HT)
7555                         break;
7556
7557                 if (i == RTE_DIM(instr->dma.dst.header_id))
7558                         break;
7559
7560                 if (i && data[i].n_users)
7561                         break;
7562         }
7563
7564         if (i < 2)
7565                 return 0;
7566
7567         *n_pattern_instr = i;
7568         return 1;
7569 }
7570
7571 static void
7572 instr_pattern_dma_many_replace(struct instruction *instr,
7573                                struct instruction_data *data,
7574                                uint32_t n_instr)
7575 {
7576         uint32_t i;
7577
7578         for (i = 1; i < n_instr; i++) {
7579                 instr[0].type++;
7580                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7581                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7582                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7583                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7584
7585                 data[i].invalid = 1;
7586         }
7587 }
7588
7589 static uint32_t
7590 instr_pattern_dma_many_optimize(struct instruction *instructions,
7591                struct instruction_data *instruction_data,
7592                uint32_t n_instructions)
7593 {
7594         uint32_t i;
7595
7596         for (i = 0; i < n_instructions; ) {
7597                 struct instruction *instr = &instructions[i];
7598                 struct instruction_data *data = &instruction_data[i];
7599                 uint32_t n_instr = 0;
7600                 int detected;
7601
7602                 /* DMA many. */
7603                 detected = instr_pattern_dma_many_search(instr,
7604                                                          data,
7605                                                          n_instructions - i,
7606                                                          &n_instr);
7607                 if (detected) {
7608                         instr_pattern_dma_many_replace(instr, data, n_instr);
7609                         i += n_instr;
7610                         continue;
7611                 }
7612
7613                 /* No pattern starting at the current instruction. */
7614                 i++;
7615         }
7616
7617         /* Eliminate the invalid instructions that have been optimized out. */
7618         n_instructions = instr_compact(instructions,
7619                                        instruction_data,
7620                                        n_instructions);
7621
7622         return n_instructions;
7623 }
7624
7625 static uint32_t
7626 instr_optimize(struct rte_swx_pipeline *p,
7627                struct action *a,
7628                struct instruction *instructions,
7629                struct instruction_data *instruction_data,
7630                uint32_t n_instructions)
7631 {
7632         /* Extract many. */
7633         n_instructions = instr_pattern_extract_many_optimize(instructions,
7634                                                              instruction_data,
7635                                                              n_instructions);
7636
7637         /* Emit many + TX. */
7638         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7639                                                              instruction_data,
7640                                                              n_instructions);
7641
7642         /* Mov all + validate. */
7643         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7644                                                                  a,
7645                                                                  instructions,
7646                                                                  instruction_data,
7647                                                                  n_instructions);
7648
7649         /* DMA many. */
7650         n_instructions = instr_pattern_dma_many_optimize(instructions,
7651                                                          instruction_data,
7652                                                          n_instructions);
7653
7654         return n_instructions;
7655 }
7656
7657 static int
7658 instruction_config(struct rte_swx_pipeline *p,
7659                    struct action *a,
7660                    const char **instructions,
7661                    uint32_t n_instructions)
7662 {
7663         struct instruction *instr = NULL;
7664         struct instruction_data *data = NULL;
7665         int err = 0;
7666         uint32_t i;
7667
7668         CHECK(n_instructions, EINVAL);
7669         CHECK(instructions, EINVAL);
7670         for (i = 0; i < n_instructions; i++)
7671                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7672
7673         /* Memory allocation. */
7674         instr = calloc(n_instructions, sizeof(struct instruction));
7675         if (!instr) {
7676                 err = -ENOMEM;
7677                 goto error;
7678         }
7679
7680         data = calloc(n_instructions, sizeof(struct instruction_data));
7681         if (!data) {
7682                 err = -ENOMEM;
7683                 goto error;
7684         }
7685
7686         for (i = 0; i < n_instructions; i++) {
7687                 char *string = strdup(instructions[i]);
7688                 if (!string) {
7689                         err = -ENOMEM;
7690                         goto error;
7691                 }
7692
7693                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7694                 if (err) {
7695                         free(string);
7696                         goto error;
7697                 }
7698
7699                 free(string);
7700         }
7701
7702         err = instr_label_check(data, n_instructions);
7703         if (err)
7704                 goto error;
7705
7706         err = instr_verify(p, a, instr, data, n_instructions);
7707         if (err)
7708                 goto error;
7709
7710         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7711
7712         err = instr_jmp_resolve(instr, data, n_instructions);
7713         if (err)
7714                 goto error;
7715
7716         if (a) {
7717                 a->instructions = instr;
7718                 a->n_instructions = n_instructions;
7719         } else {
7720                 p->instructions = instr;
7721                 p->n_instructions = n_instructions;
7722         }
7723
7724         free(data);
7725         return 0;
7726
7727 error:
7728         free(data);
7729         free(instr);
7730         return err;
7731 }
7732
7733 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7734
7735 static instr_exec_t instruction_table[] = {
7736         [INSTR_RX] = instr_rx_exec,
7737         [INSTR_TX] = instr_tx_exec,
7738         [INSTR_TX_I] = instr_tx_i_exec,
7739
7740         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7741         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7742         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7743         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7744         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7745         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7746         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7747         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7748         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7749         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7750
7751         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7752         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7753         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7754         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7755         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7756         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7757         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7758         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7759         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7760
7761         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7762         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7763
7764         [INSTR_MOV] = instr_mov_exec,
7765         [INSTR_MOV_MH] = instr_mov_mh_exec,
7766         [INSTR_MOV_HM] = instr_mov_hm_exec,
7767         [INSTR_MOV_HH] = instr_mov_hh_exec,
7768         [INSTR_MOV_I] = instr_mov_i_exec,
7769
7770         [INSTR_DMA_HT] = instr_dma_ht_exec,
7771         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7772         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7773         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7774         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7775         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7776         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7777         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7778
7779         [INSTR_ALU_ADD] = instr_alu_add_exec,
7780         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7781         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7782         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7783         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7784         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7785
7786         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7787         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7788         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7789         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7790         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7791         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7792
7793         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7794         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7795         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7796         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7797
7798         [INSTR_ALU_AND] = instr_alu_and_exec,
7799         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7800         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7801         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7802         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7803
7804         [INSTR_ALU_OR] = instr_alu_or_exec,
7805         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7806         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7807         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7808         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7809
7810         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7811         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7812         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7813         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7814         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7815
7816         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7817         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7818         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7819         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7820         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7821         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7822
7823         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7824         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7825         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7826         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7827         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7828         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7829
7830         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7831         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7832         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7833
7834         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7835         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7836         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7837         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7838         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7839         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7840
7841         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7842         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7843         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7844         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7845         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7846         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7847         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7848         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7849         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7850
7851         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7852         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7853         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7854         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7855         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7856         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7857         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7858         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7859         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7860
7861         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7862         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7863         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7864
7865         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7866         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7867         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7868         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7869         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7870         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7871         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7872         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7873         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7874         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7875         [INSTR_METER_IMM] = instr_meter_imm_exec,
7876         [INSTR_METER_IMI] = instr_meter_imi_exec,
7877
7878         [INSTR_TABLE] = instr_table_exec,
7879         [INSTR_SELECTOR] = instr_selector_exec,
7880         [INSTR_LEARNER] = instr_learner_exec,
7881         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7882         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7883         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7884         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7885
7886         [INSTR_JMP] = instr_jmp_exec,
7887         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7888         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7889         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7890         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7891         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7892         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7893
7894         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7895         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7896         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7897         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7898         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7899
7900         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7901         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7902         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7903         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7904         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7905
7906         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7907         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7908         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7909         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7910         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7911         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7912
7913         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7914         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7915         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7916         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7917         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7918         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7919
7920         [INSTR_RETURN] = instr_return_exec,
7921 };
7922
7923 static inline void
7924 instr_exec(struct rte_swx_pipeline *p)
7925 {
7926         struct thread *t = &p->threads[p->thread_id];
7927         struct instruction *ip = t->ip;
7928         instr_exec_t instr = instruction_table[ip->type];
7929
7930         instr(p);
7931 }
7932
7933 /*
7934  * Action.
7935  */
7936 static struct action *
7937 action_find(struct rte_swx_pipeline *p, const char *name)
7938 {
7939         struct action *elem;
7940
7941         if (!name)
7942                 return NULL;
7943
7944         TAILQ_FOREACH(elem, &p->actions, node)
7945                 if (strcmp(elem->name, name) == 0)
7946                         return elem;
7947
7948         return NULL;
7949 }
7950
7951 static struct action *
7952 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7953 {
7954         struct action *action = NULL;
7955
7956         TAILQ_FOREACH(action, &p->actions, node)
7957                 if (action->id == id)
7958                         return action;
7959
7960         return NULL;
7961 }
7962
7963 static struct field *
7964 action_field_find(struct action *a, const char *name)
7965 {
7966         return a->st ? struct_type_field_find(a->st, name) : NULL;
7967 }
7968
7969 static struct field *
7970 action_field_parse(struct action *action, const char *name)
7971 {
7972         if (name[0] != 't' || name[1] != '.')
7973                 return NULL;
7974
7975         return action_field_find(action, &name[2]);
7976 }
7977
7978 static int
7979 action_has_nbo_args(struct action *a)
7980 {
7981         uint32_t i;
7982
7983         /* Return if the action does not have any args. */
7984         if (!a->st)
7985                 return 0; /* FALSE */
7986
7987         for (i = 0; i < a->st->n_fields; i++)
7988                 if (a->args_endianness[i])
7989                         return 1; /* TRUE */
7990
7991         return 0; /* FALSE */
7992 }
7993
7994 static int
7995 action_does_learning(struct action *a)
7996 {
7997         uint32_t i;
7998
7999         for (i = 0; i < a->n_instructions; i++)
8000                 switch (a->instructions[i].type) {
8001                 case INSTR_LEARNER_LEARN:
8002                         return 1; /* TRUE */
8003
8004                 case INSTR_LEARNER_FORGET:
8005                         return 1; /* TRUE */
8006
8007                 default:
8008                         continue;
8009                 }
8010
8011         return 0; /* FALSE */
8012 }
8013
8014 int
8015 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8016                                const char *name,
8017                                const char *args_struct_type_name,
8018                                const char **instructions,
8019                                uint32_t n_instructions)
8020 {
8021         struct struct_type *args_struct_type = NULL;
8022         struct action *a;
8023         int err;
8024
8025         CHECK(p, EINVAL);
8026
8027         CHECK_NAME(name, EINVAL);
8028         CHECK(!action_find(p, name), EEXIST);
8029
8030         if (args_struct_type_name) {
8031                 CHECK_NAME(args_struct_type_name, EINVAL);
8032                 args_struct_type = struct_type_find(p, args_struct_type_name);
8033                 CHECK(args_struct_type, EINVAL);
8034                 CHECK(!args_struct_type->var_size, EINVAL);
8035         }
8036
8037         /* Node allocation. */
8038         a = calloc(1, sizeof(struct action));
8039         CHECK(a, ENOMEM);
8040         if (args_struct_type) {
8041                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
8042                 if (!a->args_endianness) {
8043                         free(a);
8044                         CHECK(0, ENOMEM);
8045                 }
8046         }
8047
8048         /* Node initialization. */
8049         strcpy(a->name, name);
8050         a->st = args_struct_type;
8051         a->id = p->n_actions;
8052
8053         /* Instruction translation. */
8054         err = instruction_config(p, a, instructions, n_instructions);
8055         if (err) {
8056                 free(a->args_endianness);
8057                 free(a);
8058                 return err;
8059         }
8060
8061         /* Node add to tailq. */
8062         TAILQ_INSERT_TAIL(&p->actions, a, node);
8063         p->n_actions++;
8064
8065         return 0;
8066 }
8067
8068 static int
8069 action_build(struct rte_swx_pipeline *p)
8070 {
8071         struct action *action;
8072
8073         p->action_instructions = calloc(p->n_actions,
8074                                         sizeof(struct instruction *));
8075         CHECK(p->action_instructions, ENOMEM);
8076
8077         TAILQ_FOREACH(action, &p->actions, node)
8078                 p->action_instructions[action->id] = action->instructions;
8079
8080         return 0;
8081 }
8082
8083 static void
8084 action_build_free(struct rte_swx_pipeline *p)
8085 {
8086         free(p->action_instructions);
8087         p->action_instructions = NULL;
8088 }
8089
8090 static void
8091 action_free(struct rte_swx_pipeline *p)
8092 {
8093         action_build_free(p);
8094
8095         for ( ; ; ) {
8096                 struct action *action;
8097
8098                 action = TAILQ_FIRST(&p->actions);
8099                 if (!action)
8100                         break;
8101
8102                 TAILQ_REMOVE(&p->actions, action, node);
8103                 free(action->instructions);
8104                 free(action);
8105         }
8106 }
8107
8108 static uint32_t
8109 action_arg_src_mov_count(struct action *a,
8110                          uint32_t arg_id,
8111                          struct instruction *instructions,
8112                          struct instruction_data *instruction_data,
8113                          uint32_t n_instructions)
8114 {
8115         uint32_t offset, n_users = 0, i;
8116
8117         if (!a->st ||
8118             (arg_id >= a->st->n_fields) ||
8119             !instructions ||
8120             !instruction_data ||
8121             !n_instructions)
8122                 return 0;
8123
8124         offset = a->st->fields[arg_id].offset / 8;
8125
8126         for (i = 0; i < n_instructions; i++) {
8127                 struct instruction *instr = &instructions[i];
8128                 struct instruction_data *data = &instruction_data[i];
8129
8130                 if (data->invalid ||
8131                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
8132                     instr->mov.src.struct_id ||
8133                     (instr->mov.src.offset != offset))
8134                         continue;
8135
8136                 n_users++;
8137         }
8138
8139         return n_users;
8140 }
8141
8142 /*
8143  * Table.
8144  */
8145 static struct table_type *
8146 table_type_find(struct rte_swx_pipeline *p, const char *name)
8147 {
8148         struct table_type *elem;
8149
8150         TAILQ_FOREACH(elem, &p->table_types, node)
8151                 if (strcmp(elem->name, name) == 0)
8152                         return elem;
8153
8154         return NULL;
8155 }
8156
8157 static struct table_type *
8158 table_type_resolve(struct rte_swx_pipeline *p,
8159                    const char *recommended_type_name,
8160                    enum rte_swx_table_match_type match_type)
8161 {
8162         struct table_type *elem;
8163
8164         /* Only consider the recommended type if the match type is correct. */
8165         if (recommended_type_name)
8166                 TAILQ_FOREACH(elem, &p->table_types, node)
8167                         if (!strcmp(elem->name, recommended_type_name) &&
8168                             (elem->match_type == match_type))
8169                                 return elem;
8170
8171         /* Ignore the recommended type and get the first element with this match
8172          * type.
8173          */
8174         TAILQ_FOREACH(elem, &p->table_types, node)
8175                 if (elem->match_type == match_type)
8176                         return elem;
8177
8178         return NULL;
8179 }
8180
8181 static struct table *
8182 table_find(struct rte_swx_pipeline *p, const char *name)
8183 {
8184         struct table *elem;
8185
8186         TAILQ_FOREACH(elem, &p->tables, node)
8187                 if (strcmp(elem->name, name) == 0)
8188                         return elem;
8189
8190         return NULL;
8191 }
8192
8193 static struct table *
8194 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8195 {
8196         struct table *table = NULL;
8197
8198         TAILQ_FOREACH(table, &p->tables, node)
8199                 if (table->id == id)
8200                         return table;
8201
8202         return NULL;
8203 }
8204
8205 int
8206 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8207                                      const char *name,
8208                                      enum rte_swx_table_match_type match_type,
8209                                      struct rte_swx_table_ops *ops)
8210 {
8211         struct table_type *elem;
8212
8213         CHECK(p, EINVAL);
8214
8215         CHECK_NAME(name, EINVAL);
8216         CHECK(!table_type_find(p, name), EEXIST);
8217
8218         CHECK(ops, EINVAL);
8219         CHECK(ops->create, EINVAL);
8220         CHECK(ops->lkp, EINVAL);
8221         CHECK(ops->free, EINVAL);
8222
8223         /* Node allocation. */
8224         elem = calloc(1, sizeof(struct table_type));
8225         CHECK(elem, ENOMEM);
8226
8227         /* Node initialization. */
8228         strcpy(elem->name, name);
8229         elem->match_type = match_type;
8230         memcpy(&elem->ops, ops, sizeof(*ops));
8231
8232         /* Node add to tailq. */
8233         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8234
8235         return 0;
8236 }
8237
8238 static int
8239 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8240                          uint32_t n_fields,
8241                          enum rte_swx_table_match_type *match_type)
8242 {
8243         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
8244
8245         for (i = 0; i < n_fields; i++) {
8246                 struct rte_swx_match_field_params  *f = &fields[i];
8247
8248                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
8249                         n_fields_em++;
8250
8251                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
8252                         n_fields_lpm++;
8253         }
8254
8255         if ((n_fields_lpm > 1) ||
8256             (n_fields_lpm && (n_fields_em != n_fields - 1)))
8257                 return -EINVAL;
8258
8259         *match_type = (n_fields_em == n_fields) ?
8260                        RTE_SWX_TABLE_MATCH_EXACT :
8261                        RTE_SWX_TABLE_MATCH_WILDCARD;
8262
8263         return 0;
8264 }
8265
8266 static int
8267 table_match_fields_check(struct rte_swx_pipeline *p,
8268                          struct rte_swx_pipeline_table_params *params,
8269                          struct header **header)
8270 {
8271         struct header *h0 = NULL;
8272         struct field *hf, *mf;
8273         uint32_t *offset = NULL, i;
8274         int status = 0;
8275
8276         /* Return if no match fields. */
8277         if (!params->n_fields) {
8278                 if (params->fields) {
8279                         status = -EINVAL;
8280                         goto end;
8281                 }
8282
8283                 if (header)
8284                         *header = NULL;
8285
8286                 return 0;
8287         }
8288
8289         /* Memory allocation. */
8290         offset = calloc(params->n_fields, sizeof(uint32_t));
8291         if (!offset) {
8292                 status = -ENOMEM;
8293                 goto end;
8294         }
8295
8296         /* Check that all the match fields belong to either the same header or
8297          * to the meta-data.
8298          */
8299         hf = header_field_parse(p, params->fields[0].name, &h0);
8300         mf = metadata_field_parse(p, params->fields[0].name);
8301         if ((!hf && !mf) || (hf && hf->var_size)) {
8302                 status = -EINVAL;
8303                 goto end;
8304         }
8305
8306         offset[0] = h0 ? hf->offset : mf->offset;
8307
8308         for (i = 1; i < params->n_fields; i++)
8309                 if (h0) {
8310                         struct header *h;
8311
8312                         hf = header_field_parse(p, params->fields[i].name, &h);
8313                         if (!hf || (h->id != h0->id) || hf->var_size) {
8314                                 status = -EINVAL;
8315                                 goto end;
8316                         }
8317
8318                         offset[i] = hf->offset;
8319                 } else {
8320                         mf = metadata_field_parse(p, params->fields[i].name);
8321                         if (!mf) {
8322                                 status = -EINVAL;
8323                                 goto end;
8324                         }
8325
8326                         offset[i] = mf->offset;
8327                 }
8328
8329         /* Check that there are no duplicated match fields. */
8330         for (i = 0; i < params->n_fields; i++) {
8331                 uint32_t j;
8332
8333                 for (j = 0; j < i; j++)
8334                         if (offset[j] == offset[i]) {
8335                                 status = -EINVAL;
8336                                 goto end;
8337                         }
8338         }
8339
8340         /* Return. */
8341         if (header)
8342                 *header = h0;
8343
8344 end:
8345         free(offset);
8346         return status;
8347 }
8348
8349 int
8350 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8351                               const char *name,
8352                               struct rte_swx_pipeline_table_params *params,
8353                               const char *recommended_table_type_name,
8354                               const char *args,
8355                               uint32_t size)
8356 {
8357         struct table_type *type;
8358         struct table *t;
8359         struct action *default_action;
8360         struct header *header = NULL;
8361         uint32_t action_data_size_max = 0, i;
8362         int status = 0;
8363
8364         CHECK(p, EINVAL);
8365
8366         CHECK_NAME(name, EINVAL);
8367         CHECK(!table_find(p, name), EEXIST);
8368         CHECK(!selector_find(p, name), EEXIST);
8369         CHECK(!learner_find(p, name), EEXIST);
8370
8371         CHECK(params, EINVAL);
8372
8373         /* Match checks. */
8374         status = table_match_fields_check(p, params, &header);
8375         if (status)
8376                 return status;
8377
8378         /* Action checks. */
8379         CHECK(params->n_actions, EINVAL);
8380         CHECK(params->action_names, EINVAL);
8381         for (i = 0; i < params->n_actions; i++) {
8382                 const char *action_name = params->action_names[i];
8383                 struct action *a;
8384                 uint32_t action_data_size;
8385
8386                 CHECK_NAME(action_name, EINVAL);
8387
8388                 a = action_find(p, action_name);
8389                 CHECK(a, EINVAL);
8390                 CHECK(!action_does_learning(a), EINVAL);
8391
8392                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8393                 if (action_data_size > action_data_size_max)
8394                         action_data_size_max = action_data_size;
8395         }
8396
8397         CHECK_NAME(params->default_action_name, EINVAL);
8398         for (i = 0; i < p->n_actions; i++)
8399                 if (!strcmp(params->action_names[i],
8400                             params->default_action_name))
8401                         break;
8402         CHECK(i < params->n_actions, EINVAL);
8403         default_action = action_find(p, params->default_action_name);
8404         CHECK((default_action->st && params->default_action_data) ||
8405               !params->default_action_data, EINVAL);
8406
8407         /* Table type checks. */
8408         if (recommended_table_type_name)
8409                 CHECK_NAME(recommended_table_type_name, EINVAL);
8410
8411         if (params->n_fields) {
8412                 enum rte_swx_table_match_type match_type;
8413
8414                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8415                 if (status)
8416                         return status;
8417
8418                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8419                 CHECK(type, EINVAL);
8420         } else {
8421                 type = NULL;
8422         }
8423
8424         /* Memory allocation. */
8425         t = calloc(1, sizeof(struct table));
8426         CHECK(t, ENOMEM);
8427
8428         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8429         if (!t->fields) {
8430                 free(t);
8431                 CHECK(0, ENOMEM);
8432         }
8433
8434         t->actions = calloc(params->n_actions, sizeof(struct action *));
8435         if (!t->actions) {
8436                 free(t->fields);
8437                 free(t);
8438                 CHECK(0, ENOMEM);
8439         }
8440
8441         if (action_data_size_max) {
8442                 t->default_action_data = calloc(1, action_data_size_max);
8443                 if (!t->default_action_data) {
8444                         free(t->actions);
8445                         free(t->fields);
8446                         free(t);
8447                         CHECK(0, ENOMEM);
8448                 }
8449         }
8450
8451         /* Node initialization. */
8452         strcpy(t->name, name);
8453         if (args && args[0])
8454                 strcpy(t->args, args);
8455         t->type = type;
8456
8457         for (i = 0; i < params->n_fields; i++) {
8458                 struct rte_swx_match_field_params *field = &params->fields[i];
8459                 struct match_field *f = &t->fields[i];
8460
8461                 f->match_type = field->match_type;
8462                 f->field = header ?
8463                         header_field_parse(p, field->name, NULL) :
8464                         metadata_field_parse(p, field->name);
8465         }
8466         t->n_fields = params->n_fields;
8467         t->header = header;
8468
8469         for (i = 0; i < params->n_actions; i++)
8470                 t->actions[i] = action_find(p, params->action_names[i]);
8471         t->default_action = default_action;
8472         if (default_action->st)
8473                 memcpy(t->default_action_data,
8474                        params->default_action_data,
8475                        default_action->st->n_bits / 8);
8476         t->n_actions = params->n_actions;
8477         t->default_action_is_const = params->default_action_is_const;
8478         t->action_data_size_max = action_data_size_max;
8479
8480         t->size = size;
8481         t->id = p->n_tables;
8482
8483         /* Node add to tailq. */
8484         TAILQ_INSERT_TAIL(&p->tables, t, node);
8485         p->n_tables++;
8486
8487         return 0;
8488 }
8489
8490 static struct rte_swx_table_params *
8491 table_params_get(struct table *table)
8492 {
8493         struct rte_swx_table_params *params;
8494         struct field *first, *last;
8495         uint8_t *key_mask;
8496         uint32_t key_size, key_offset, action_data_size, i;
8497
8498         /* Memory allocation. */
8499         params = calloc(1, sizeof(struct rte_swx_table_params));
8500         if (!params)
8501                 return NULL;
8502
8503         /* Find first (smallest offset) and last (biggest offset) match fields. */
8504         first = table->fields[0].field;
8505         last = table->fields[0].field;
8506
8507         for (i = 0; i < table->n_fields; i++) {
8508                 struct field *f = table->fields[i].field;
8509
8510                 if (f->offset < first->offset)
8511                         first = f;
8512
8513                 if (f->offset > last->offset)
8514                         last = f;
8515         }
8516
8517         /* Key offset and size. */
8518         key_offset = first->offset / 8;
8519         key_size = (last->offset + last->n_bits - first->offset) / 8;
8520
8521         /* Memory allocation. */
8522         key_mask = calloc(1, key_size);
8523         if (!key_mask) {
8524                 free(params);
8525                 return NULL;
8526         }
8527
8528         /* Key mask. */
8529         for (i = 0; i < table->n_fields; i++) {
8530                 struct field *f = table->fields[i].field;
8531                 uint32_t start = (f->offset - first->offset) / 8;
8532                 size_t size = f->n_bits / 8;
8533
8534                 memset(&key_mask[start], 0xFF, size);
8535         }
8536
8537         /* Action data size. */
8538         action_data_size = 0;
8539         for (i = 0; i < table->n_actions; i++) {
8540                 struct action *action = table->actions[i];
8541                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8542
8543                 if (ads > action_data_size)
8544                         action_data_size = ads;
8545         }
8546
8547         /* Fill in. */
8548         params->match_type = table->type->match_type;
8549         params->key_size = key_size;
8550         params->key_offset = key_offset;
8551         params->key_mask0 = key_mask;
8552         params->action_data_size = action_data_size;
8553         params->n_keys_max = table->size;
8554
8555         return params;
8556 }
8557
8558 static void
8559 table_params_free(struct rte_swx_table_params *params)
8560 {
8561         if (!params)
8562                 return;
8563
8564         free(params->key_mask0);
8565         free(params);
8566 }
8567
8568 static int
8569 table_stub_lkp(void *table __rte_unused,
8570                void *mailbox __rte_unused,
8571                uint8_t **key __rte_unused,
8572                uint64_t *action_id __rte_unused,
8573                uint8_t **action_data __rte_unused,
8574                int *hit)
8575 {
8576         *hit = 0;
8577         return 1; /* DONE. */
8578 }
8579
8580 static int
8581 table_build(struct rte_swx_pipeline *p)
8582 {
8583         uint32_t i;
8584
8585         /* Per pipeline: table statistics. */
8586         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8587         CHECK(p->table_stats, ENOMEM);
8588
8589         for (i = 0; i < p->n_tables; i++) {
8590                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8591                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8592         }
8593
8594         /* Per thread: table runt-time. */
8595         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8596                 struct thread *t = &p->threads[i];
8597                 struct table *table;
8598
8599                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8600                 CHECK(t->tables, ENOMEM);
8601
8602                 TAILQ_FOREACH(table, &p->tables, node) {
8603                         struct table_runtime *r = &t->tables[table->id];
8604
8605                         if (table->type) {
8606                                 uint64_t size;
8607
8608                                 size = table->type->ops.mailbox_size_get();
8609
8610                                 /* r->func. */
8611                                 r->func = table->type->ops.lkp;
8612
8613                                 /* r->mailbox. */
8614                                 if (size) {
8615                                         r->mailbox = calloc(1, size);
8616                                         CHECK(r->mailbox, ENOMEM);
8617                                 }
8618
8619                                 /* r->key. */
8620                                 r->key = table->header ?
8621                                         &t->structs[table->header->struct_id] :
8622                                         &t->structs[p->metadata_struct_id];
8623                         } else {
8624                                 r->func = table_stub_lkp;
8625                         }
8626                 }
8627         }
8628
8629         return 0;
8630 }
8631
8632 static void
8633 table_build_free(struct rte_swx_pipeline *p)
8634 {
8635         uint32_t i;
8636
8637         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8638                 struct thread *t = &p->threads[i];
8639                 uint32_t j;
8640
8641                 if (!t->tables)
8642                         continue;
8643
8644                 for (j = 0; j < p->n_tables; j++) {
8645                         struct table_runtime *r = &t->tables[j];
8646
8647                         free(r->mailbox);
8648                 }
8649
8650                 free(t->tables);
8651                 t->tables = NULL;
8652         }
8653
8654         if (p->table_stats) {
8655                 for (i = 0; i < p->n_tables; i++)
8656                         free(p->table_stats[i].n_pkts_action);
8657
8658                 free(p->table_stats);
8659         }
8660 }
8661
8662 static void
8663 table_free(struct rte_swx_pipeline *p)
8664 {
8665         table_build_free(p);
8666
8667         /* Tables. */
8668         for ( ; ; ) {
8669                 struct table *elem;
8670
8671                 elem = TAILQ_FIRST(&p->tables);
8672                 if (!elem)
8673                         break;
8674
8675                 TAILQ_REMOVE(&p->tables, elem, node);
8676                 free(elem->fields);
8677                 free(elem->actions);
8678                 free(elem->default_action_data);
8679                 free(elem);
8680         }
8681
8682         /* Table types. */
8683         for ( ; ; ) {
8684                 struct table_type *elem;
8685
8686                 elem = TAILQ_FIRST(&p->table_types);
8687                 if (!elem)
8688                         break;
8689
8690                 TAILQ_REMOVE(&p->table_types, elem, node);
8691                 free(elem);
8692         }
8693 }
8694
8695 /*
8696  * Selector.
8697  */
8698 static struct selector *
8699 selector_find(struct rte_swx_pipeline *p, const char *name)
8700 {
8701         struct selector *s;
8702
8703         TAILQ_FOREACH(s, &p->selectors, node)
8704                 if (strcmp(s->name, name) == 0)
8705                         return s;
8706
8707         return NULL;
8708 }
8709
8710 static struct selector *
8711 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8712 {
8713         struct selector *s = NULL;
8714
8715         TAILQ_FOREACH(s, &p->selectors, node)
8716                 if (s->id == id)
8717                         return s;
8718
8719         return NULL;
8720 }
8721
8722 static int
8723 selector_fields_check(struct rte_swx_pipeline *p,
8724                       struct rte_swx_pipeline_selector_params *params,
8725                       struct header **header)
8726 {
8727         struct header *h0 = NULL;
8728         struct field *hf, *mf;
8729         uint32_t i;
8730
8731         /* Return if no selector fields. */
8732         if (!params->n_selector_fields || !params->selector_field_names)
8733                 return -EINVAL;
8734
8735         /* Check that all the selector fields either belong to the same header
8736          * or are all meta-data fields.
8737          */
8738         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8739         mf = metadata_field_parse(p, params->selector_field_names[0]);
8740         if (!hf && !mf)
8741                 return -EINVAL;
8742
8743         for (i = 1; i < params->n_selector_fields; i++)
8744                 if (h0) {
8745                         struct header *h;
8746
8747                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8748                         if (!hf || (h->id != h0->id))
8749                                 return -EINVAL;
8750                 } else {
8751                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8752                         if (!mf)
8753                                 return -EINVAL;
8754                 }
8755
8756         /* Check that there are no duplicated match fields. */
8757         for (i = 0; i < params->n_selector_fields; i++) {
8758                 const char *field_name = params->selector_field_names[i];
8759                 uint32_t j;
8760
8761                 for (j = i + 1; j < params->n_selector_fields; j++)
8762                         if (!strcmp(params->selector_field_names[j], field_name))
8763                                 return -EINVAL;
8764         }
8765
8766         /* Return. */
8767         if (header)
8768                 *header = h0;
8769
8770         return 0;
8771 }
8772
8773 int
8774 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8775                                  const char *name,
8776                                  struct rte_swx_pipeline_selector_params *params)
8777 {
8778         struct selector *s;
8779         struct header *selector_header = NULL;
8780         struct field *group_id_field, *member_id_field;
8781         uint32_t i;
8782         int status = 0;
8783
8784         CHECK(p, EINVAL);
8785
8786         CHECK_NAME(name, EINVAL);
8787         CHECK(!table_find(p, name), EEXIST);
8788         CHECK(!selector_find(p, name), EEXIST);
8789         CHECK(!learner_find(p, name), EEXIST);
8790
8791         CHECK(params, EINVAL);
8792
8793         CHECK_NAME(params->group_id_field_name, EINVAL);
8794         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8795         CHECK(group_id_field, EINVAL);
8796
8797         for (i = 0; i < params->n_selector_fields; i++) {
8798                 const char *field_name = params->selector_field_names[i];
8799
8800                 CHECK_NAME(field_name, EINVAL);
8801         }
8802         status = selector_fields_check(p, params, &selector_header);
8803         if (status)
8804                 return status;
8805
8806         CHECK_NAME(params->member_id_field_name, EINVAL);
8807         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8808         CHECK(member_id_field, EINVAL);
8809
8810         CHECK(params->n_groups_max, EINVAL);
8811
8812         CHECK(params->n_members_per_group_max, EINVAL);
8813
8814         /* Memory allocation. */
8815         s = calloc(1, sizeof(struct selector));
8816         if (!s) {
8817                 status = -ENOMEM;
8818                 goto error;
8819         }
8820
8821         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8822         if (!s->selector_fields) {
8823                 status = -ENOMEM;
8824                 goto error;
8825         }
8826
8827         /* Node initialization. */
8828         strcpy(s->name, name);
8829
8830         s->group_id_field = group_id_field;
8831
8832         for (i = 0; i < params->n_selector_fields; i++) {
8833                 const char *field_name = params->selector_field_names[i];
8834
8835                 s->selector_fields[i] = selector_header ?
8836                         header_field_parse(p, field_name, NULL) :
8837                         metadata_field_parse(p, field_name);
8838         }
8839
8840         s->n_selector_fields = params->n_selector_fields;
8841
8842         s->selector_header = selector_header;
8843
8844         s->member_id_field = member_id_field;
8845
8846         s->n_groups_max = params->n_groups_max;
8847
8848         s->n_members_per_group_max = params->n_members_per_group_max;
8849
8850         s->id = p->n_selectors;
8851
8852         /* Node add to tailq. */
8853         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8854         p->n_selectors++;
8855
8856         return 0;
8857
8858 error:
8859         if (!s)
8860                 return status;
8861
8862         free(s->selector_fields);
8863
8864         free(s);
8865
8866         return status;
8867 }
8868
8869 static void
8870 selector_params_free(struct rte_swx_table_selector_params *params)
8871 {
8872         if (!params)
8873                 return;
8874
8875         free(params->selector_mask);
8876
8877         free(params);
8878 }
8879
8880 static struct rte_swx_table_selector_params *
8881 selector_table_params_get(struct selector *s)
8882 {
8883         struct rte_swx_table_selector_params *params = NULL;
8884         struct field *first, *last;
8885         uint32_t i;
8886
8887         /* Memory allocation. */
8888         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8889         if (!params)
8890                 goto error;
8891
8892         /* Group ID. */
8893         params->group_id_offset = s->group_id_field->offset / 8;
8894
8895         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8896         first = s->selector_fields[0];
8897         last = s->selector_fields[0];
8898
8899         for (i = 0; i < s->n_selector_fields; i++) {
8900                 struct field *f = s->selector_fields[i];
8901
8902                 if (f->offset < first->offset)
8903                         first = f;
8904
8905                 if (f->offset > last->offset)
8906                         last = f;
8907         }
8908
8909         /* Selector offset and size. */
8910         params->selector_offset = first->offset / 8;
8911         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8912
8913         /* Memory allocation. */
8914         params->selector_mask = calloc(1, params->selector_size);
8915         if (!params->selector_mask)
8916                 goto error;
8917
8918         /* Selector mask. */
8919         for (i = 0; i < s->n_selector_fields; i++) {
8920                 struct field *f = s->selector_fields[i];
8921                 uint32_t start = (f->offset - first->offset) / 8;
8922                 size_t size = f->n_bits / 8;
8923
8924                 memset(&params->selector_mask[start], 0xFF, size);
8925         }
8926
8927         /* Member ID. */
8928         params->member_id_offset = s->member_id_field->offset / 8;
8929
8930         /* Maximum number of groups. */
8931         params->n_groups_max = s->n_groups_max;
8932
8933         /* Maximum number of members per group. */
8934         params->n_members_per_group_max = s->n_members_per_group_max;
8935
8936         return params;
8937
8938 error:
8939         selector_params_free(params);
8940         return NULL;
8941 }
8942
8943 static void
8944 selector_build_free(struct rte_swx_pipeline *p)
8945 {
8946         uint32_t i;
8947
8948         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8949                 struct thread *t = &p->threads[i];
8950                 uint32_t j;
8951
8952                 if (!t->selectors)
8953                         continue;
8954
8955                 for (j = 0; j < p->n_selectors; j++) {
8956                         struct selector_runtime *r = &t->selectors[j];
8957
8958                         free(r->mailbox);
8959                 }
8960
8961                 free(t->selectors);
8962                 t->selectors = NULL;
8963         }
8964
8965         free(p->selector_stats);
8966         p->selector_stats = NULL;
8967 }
8968
8969 static int
8970 selector_build(struct rte_swx_pipeline *p)
8971 {
8972         uint32_t i;
8973         int status = 0;
8974
8975         /* Per pipeline: selector statistics. */
8976         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8977         if (!p->selector_stats) {
8978                 status = -ENOMEM;
8979                 goto error;
8980         }
8981
8982         /* Per thread: selector run-time. */
8983         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8984                 struct thread *t = &p->threads[i];
8985                 struct selector *s;
8986
8987                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8988                 if (!t->selectors) {
8989                         status = -ENOMEM;
8990                         goto error;
8991                 }
8992
8993                 TAILQ_FOREACH(s, &p->selectors, node) {
8994                         struct selector_runtime *r = &t->selectors[s->id];
8995                         uint64_t size;
8996
8997                         /* r->mailbox. */
8998                         size = rte_swx_table_selector_mailbox_size_get();
8999                         if (size) {
9000                                 r->mailbox = calloc(1, size);
9001                                 if (!r->mailbox) {
9002                                         status = -ENOMEM;
9003                                         goto error;
9004                                 }
9005                         }
9006
9007                         /* r->group_id_buffer. */
9008                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
9009
9010                         /* r->selector_buffer. */
9011                         r->selector_buffer = s->selector_header ?
9012                                 &t->structs[s->selector_header->struct_id] :
9013                                 &t->structs[p->metadata_struct_id];
9014
9015                         /* r->member_id_buffer. */
9016                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
9017                 }
9018         }
9019
9020         return 0;
9021
9022 error:
9023         selector_build_free(p);
9024         return status;
9025 }
9026
9027 static void
9028 selector_free(struct rte_swx_pipeline *p)
9029 {
9030         selector_build_free(p);
9031
9032         /* Selector tables. */
9033         for ( ; ; ) {
9034                 struct selector *elem;
9035
9036                 elem = TAILQ_FIRST(&p->selectors);
9037                 if (!elem)
9038                         break;
9039
9040                 TAILQ_REMOVE(&p->selectors, elem, node);
9041                 free(elem->selector_fields);
9042                 free(elem);
9043         }
9044 }
9045
9046 /*
9047  * Learner table.
9048  */
9049 static struct learner *
9050 learner_find(struct rte_swx_pipeline *p, const char *name)
9051 {
9052         struct learner *l;
9053
9054         TAILQ_FOREACH(l, &p->learners, node)
9055                 if (!strcmp(l->name, name))
9056                         return l;
9057
9058         return NULL;
9059 }
9060
9061 static struct learner *
9062 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9063 {
9064         struct learner *l = NULL;
9065
9066         TAILQ_FOREACH(l, &p->learners, node)
9067                 if (l->id == id)
9068                         return l;
9069
9070         return NULL;
9071 }
9072
9073 static int
9074 learner_match_fields_check(struct rte_swx_pipeline *p,
9075                            struct rte_swx_pipeline_learner_params *params,
9076                            struct header **header)
9077 {
9078         struct header *h0 = NULL;
9079         struct field *hf, *mf;
9080         uint32_t i;
9081
9082         /* Return if no match fields. */
9083         if (!params->n_fields || !params->field_names)
9084                 return -EINVAL;
9085
9086         /* Check that all the match fields either belong to the same header
9087          * or are all meta-data fields.
9088          */
9089         hf = header_field_parse(p, params->field_names[0], &h0);
9090         mf = metadata_field_parse(p, params->field_names[0]);
9091         if (!hf && !mf)
9092                 return -EINVAL;
9093
9094         for (i = 1; i < params->n_fields; i++)
9095                 if (h0) {
9096                         struct header *h;
9097
9098                         hf = header_field_parse(p, params->field_names[i], &h);
9099                         if (!hf || (h->id != h0->id))
9100                                 return -EINVAL;
9101                 } else {
9102                         mf = metadata_field_parse(p, params->field_names[i]);
9103                         if (!mf)
9104                                 return -EINVAL;
9105                 }
9106
9107         /* Check that there are no duplicated match fields. */
9108         for (i = 0; i < params->n_fields; i++) {
9109                 const char *field_name = params->field_names[i];
9110                 uint32_t j;
9111
9112                 for (j = i + 1; j < params->n_fields; j++)
9113                         if (!strcmp(params->field_names[j], field_name))
9114                                 return -EINVAL;
9115         }
9116
9117         /* Return. */
9118         if (header)
9119                 *header = h0;
9120
9121         return 0;
9122 }
9123
9124 static int
9125 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
9126 {
9127         struct struct_type *mst = p->metadata_st, *ast = a->st;
9128         struct field *mf, *af;
9129         uint32_t mf_pos, i;
9130
9131         if (!ast) {
9132                 if (mf_name)
9133                         return -EINVAL;
9134
9135                 return 0;
9136         }
9137
9138         /* Check that mf_name is the name of a valid meta-data field. */
9139         CHECK_NAME(mf_name, EINVAL);
9140         mf = metadata_field_parse(p, mf_name);
9141         CHECK(mf, EINVAL);
9142
9143         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
9144          * all the action arguments.
9145          */
9146         mf_pos = mf - mst->fields;
9147         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
9148
9149         /* Check that the size of each of the identified meta-data fields matches exactly the size
9150          * of the corresponding action argument.
9151          */
9152         for (i = 0; i < ast->n_fields; i++) {
9153                 mf = &mst->fields[mf_pos + i];
9154                 af = &ast->fields[i];
9155
9156                 CHECK(mf->n_bits == af->n_bits, EINVAL);
9157         }
9158
9159         return 0;
9160 }
9161
9162 static int
9163 learner_action_learning_check(struct rte_swx_pipeline *p,
9164                               struct action *action,
9165                               const char **action_names,
9166                               uint32_t n_actions)
9167 {
9168         uint32_t i;
9169
9170         /* For each "learn" instruction of the current action, check that the learned action (i.e.
9171          * the action passed as argument to the "learn" instruction) is also enabled for the
9172          * current learner table.
9173          */
9174         for (i = 0; i < action->n_instructions; i++) {
9175                 struct instruction *instr = &action->instructions[i];
9176                 uint32_t found = 0, j;
9177
9178                 if (instr->type != INSTR_LEARNER_LEARN)
9179                         continue;
9180
9181                 for (j = 0; j < n_actions; j++) {
9182                         struct action *a;
9183
9184                         a = action_find(p, action_names[j]);
9185                         if (!a)
9186                                 return -EINVAL;
9187
9188                         if (a->id == instr->learn.action_id)
9189                                 found = 1;
9190                 }
9191
9192                 if (!found)
9193                         return -EINVAL;
9194         }
9195
9196         return 0;
9197 }
9198
9199 int
9200 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
9201                               const char *name,
9202                               struct rte_swx_pipeline_learner_params *params,
9203                               uint32_t size,
9204                               uint32_t timeout)
9205 {
9206         struct learner *l = NULL;
9207         struct action *default_action;
9208         struct header *header = NULL;
9209         uint32_t action_data_size_max = 0, i;
9210         int status = 0;
9211
9212         CHECK(p, EINVAL);
9213
9214         CHECK_NAME(name, EINVAL);
9215         CHECK(!table_find(p, name), EEXIST);
9216         CHECK(!selector_find(p, name), EEXIST);
9217         CHECK(!learner_find(p, name), EEXIST);
9218
9219         CHECK(params, EINVAL);
9220
9221         /* Match checks. */
9222         status = learner_match_fields_check(p, params, &header);
9223         if (status)
9224                 return status;
9225
9226         /* Action checks. */
9227         CHECK(params->n_actions, EINVAL);
9228
9229         CHECK(params->action_names, EINVAL);
9230         for (i = 0; i < params->n_actions; i++) {
9231                 const char *action_name = params->action_names[i];
9232                 const char *action_field_name = params->action_field_names[i];
9233                 struct action *a;
9234                 uint32_t action_data_size;
9235
9236                 CHECK_NAME(action_name, EINVAL);
9237
9238                 a = action_find(p, action_name);
9239                 CHECK(a, EINVAL);
9240
9241                 status = learner_action_args_check(p, a, action_field_name);
9242                 if (status)
9243                         return status;
9244
9245                 status = learner_action_learning_check(p,
9246                                                        a,
9247                                                        params->action_names,
9248                                                        params->n_actions);
9249                 if (status)
9250                         return status;
9251
9252                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9253                 if (action_data_size > action_data_size_max)
9254                         action_data_size_max = action_data_size;
9255         }
9256
9257         CHECK_NAME(params->default_action_name, EINVAL);
9258         for (i = 0; i < p->n_actions; i++)
9259                 if (!strcmp(params->action_names[i],
9260                             params->default_action_name))
9261                         break;
9262         CHECK(i < params->n_actions, EINVAL);
9263
9264         default_action = action_find(p, params->default_action_name);
9265         CHECK((default_action->st && params->default_action_data) ||
9266               !params->default_action_data, EINVAL);
9267
9268         /* Any other checks. */
9269         CHECK(size, EINVAL);
9270         CHECK(timeout, EINVAL);
9271
9272         /* Memory allocation. */
9273         l = calloc(1, sizeof(struct learner));
9274         if (!l)
9275                 goto nomem;
9276
9277         l->fields = calloc(params->n_fields, sizeof(struct field *));
9278         if (!l->fields)
9279                 goto nomem;
9280
9281         l->actions = calloc(params->n_actions, sizeof(struct action *));
9282         if (!l->actions)
9283                 goto nomem;
9284
9285         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9286         if (!l->action_arg)
9287                 goto nomem;
9288
9289         if (action_data_size_max) {
9290                 l->default_action_data = calloc(1, action_data_size_max);
9291                 if (!l->default_action_data)
9292                         goto nomem;
9293         }
9294
9295         /* Node initialization. */
9296         strcpy(l->name, name);
9297
9298         for (i = 0; i < params->n_fields; i++) {
9299                 const char *field_name = params->field_names[i];
9300
9301                 l->fields[i] = header ?
9302                         header_field_parse(p, field_name, NULL) :
9303                         metadata_field_parse(p, field_name);
9304         }
9305
9306         l->n_fields = params->n_fields;
9307
9308         l->header = header;
9309
9310         for (i = 0; i < params->n_actions; i++) {
9311                 const char *mf_name = params->action_field_names[i];
9312
9313                 l->actions[i] = action_find(p, params->action_names[i]);
9314
9315                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9316         }
9317
9318         l->default_action = default_action;
9319
9320         if (default_action->st)
9321                 memcpy(l->default_action_data,
9322                        params->default_action_data,
9323                        default_action->st->n_bits / 8);
9324
9325         l->n_actions = params->n_actions;
9326
9327         l->default_action_is_const = params->default_action_is_const;
9328
9329         l->action_data_size_max = action_data_size_max;
9330
9331         l->size = size;
9332
9333         l->timeout = timeout;
9334
9335         l->id = p->n_learners;
9336
9337         /* Node add to tailq. */
9338         TAILQ_INSERT_TAIL(&p->learners, l, node);
9339         p->n_learners++;
9340
9341         return 0;
9342
9343 nomem:
9344         if (!l)
9345                 return -ENOMEM;
9346
9347         free(l->action_arg);
9348         free(l->actions);
9349         free(l->fields);
9350         free(l);
9351
9352         return -ENOMEM;
9353 }
9354
9355 static void
9356 learner_params_free(struct rte_swx_table_learner_params *params)
9357 {
9358         if (!params)
9359                 return;
9360
9361         free(params->key_mask0);
9362
9363         free(params);
9364 }
9365
9366 static struct rte_swx_table_learner_params *
9367 learner_params_get(struct learner *l)
9368 {
9369         struct rte_swx_table_learner_params *params = NULL;
9370         struct field *first, *last;
9371         uint32_t i;
9372
9373         /* Memory allocation. */
9374         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9375         if (!params)
9376                 goto error;
9377
9378         /* Find first (smallest offset) and last (biggest offset) match fields. */
9379         first = l->fields[0];
9380         last = l->fields[0];
9381
9382         for (i = 0; i < l->n_fields; i++) {
9383                 struct field *f = l->fields[i];
9384
9385                 if (f->offset < first->offset)
9386                         first = f;
9387
9388                 if (f->offset > last->offset)
9389                         last = f;
9390         }
9391
9392         /* Key offset and size. */
9393         params->key_offset = first->offset / 8;
9394         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9395
9396         /* Memory allocation. */
9397         params->key_mask0 = calloc(1, params->key_size);
9398         if (!params->key_mask0)
9399                 goto error;
9400
9401         /* Key mask. */
9402         for (i = 0; i < l->n_fields; i++) {
9403                 struct field *f = l->fields[i];
9404                 uint32_t start = (f->offset - first->offset) / 8;
9405                 size_t size = f->n_bits / 8;
9406
9407                 memset(&params->key_mask0[start], 0xFF, size);
9408         }
9409
9410         /* Action data size. */
9411         params->action_data_size = l->action_data_size_max;
9412
9413         /* Maximum number of keys. */
9414         params->n_keys_max = l->size;
9415
9416         /* Timeout. */
9417         params->key_timeout = l->timeout;
9418
9419         return params;
9420
9421 error:
9422         learner_params_free(params);
9423         return NULL;
9424 }
9425
9426 static void
9427 learner_build_free(struct rte_swx_pipeline *p)
9428 {
9429         uint32_t i;
9430
9431         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9432                 struct thread *t = &p->threads[i];
9433                 uint32_t j;
9434
9435                 if (!t->learners)
9436                         continue;
9437
9438                 for (j = 0; j < p->n_learners; j++) {
9439                         struct learner_runtime *r = &t->learners[j];
9440
9441                         free(r->mailbox);
9442                         free(r->action_data);
9443                 }
9444
9445                 free(t->learners);
9446                 t->learners = NULL;
9447         }
9448
9449         if (p->learner_stats) {
9450                 for (i = 0; i < p->n_learners; i++)
9451                         free(p->learner_stats[i].n_pkts_action);
9452
9453                 free(p->learner_stats);
9454         }
9455 }
9456
9457 static int
9458 learner_build(struct rte_swx_pipeline *p)
9459 {
9460         uint32_t i;
9461         int status = 0;
9462
9463         /* Per pipeline: learner statistics. */
9464         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9465         CHECK(p->learner_stats, ENOMEM);
9466
9467         for (i = 0; i < p->n_learners; i++) {
9468                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9469                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9470         }
9471
9472         /* Per thread: learner run-time. */
9473         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9474                 struct thread *t = &p->threads[i];
9475                 struct learner *l;
9476
9477                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9478                 if (!t->learners) {
9479                         status = -ENOMEM;
9480                         goto error;
9481                 }
9482
9483                 TAILQ_FOREACH(l, &p->learners, node) {
9484                         struct learner_runtime *r = &t->learners[l->id];
9485                         uint64_t size;
9486                         uint32_t j;
9487
9488                         /* r->mailbox. */
9489                         size = rte_swx_table_learner_mailbox_size_get();
9490                         if (size) {
9491                                 r->mailbox = calloc(1, size);
9492                                 if (!r->mailbox) {
9493                                         status = -ENOMEM;
9494                                         goto error;
9495                                 }
9496                         }
9497
9498                         /* r->key. */
9499                         r->key = l->header ?
9500                                 &t->structs[l->header->struct_id] :
9501                                 &t->structs[p->metadata_struct_id];
9502
9503                         /* r->action_data. */
9504                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9505                         if (!r->action_data) {
9506                                 status = -ENOMEM;
9507                                 goto error;
9508                         }
9509
9510                         for (j = 0; j < l->n_actions; j++) {
9511                                 struct action *a = l->actions[j];
9512                                 struct field *mf = l->action_arg[j];
9513                                 uint8_t *m = t->structs[p->metadata_struct_id];
9514
9515                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9516                         }
9517                 }
9518         }
9519
9520         return 0;
9521
9522 error:
9523         learner_build_free(p);
9524         return status;
9525 }
9526
9527 static void
9528 learner_free(struct rte_swx_pipeline *p)
9529 {
9530         learner_build_free(p);
9531
9532         /* Learner tables. */
9533         for ( ; ; ) {
9534                 struct learner *l;
9535
9536                 l = TAILQ_FIRST(&p->learners);
9537                 if (!l)
9538                         break;
9539
9540                 TAILQ_REMOVE(&p->learners, l, node);
9541                 free(l->fields);
9542                 free(l->actions);
9543                 free(l->action_arg);
9544                 free(l->default_action_data);
9545                 free(l);
9546         }
9547 }
9548
9549 /*
9550  * Table state.
9551  */
9552 static int
9553 table_state_build(struct rte_swx_pipeline *p)
9554 {
9555         struct table *table;
9556         struct selector *s;
9557         struct learner *l;
9558
9559         p->table_state = calloc(p->n_tables + p->n_selectors,
9560                                 sizeof(struct rte_swx_table_state));
9561         CHECK(p->table_state, ENOMEM);
9562
9563         TAILQ_FOREACH(table, &p->tables, node) {
9564                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9565
9566                 if (table->type) {
9567                         struct rte_swx_table_params *params;
9568
9569                         /* ts->obj. */
9570                         params = table_params_get(table);
9571                         CHECK(params, ENOMEM);
9572
9573                         ts->obj = table->type->ops.create(params,
9574                                 NULL,
9575                                 table->args,
9576                                 p->numa_node);
9577
9578                         table_params_free(params);
9579                         CHECK(ts->obj, ENODEV);
9580                 }
9581
9582                 /* ts->default_action_data. */
9583                 if (table->action_data_size_max) {
9584                         ts->default_action_data =
9585                                 malloc(table->action_data_size_max);
9586                         CHECK(ts->default_action_data, ENOMEM);
9587
9588                         memcpy(ts->default_action_data,
9589                                table->default_action_data,
9590                                table->action_data_size_max);
9591                 }
9592
9593                 /* ts->default_action_id. */
9594                 ts->default_action_id = table->default_action->id;
9595         }
9596
9597         TAILQ_FOREACH(s, &p->selectors, node) {
9598                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9599                 struct rte_swx_table_selector_params *params;
9600
9601                 /* ts->obj. */
9602                 params = selector_table_params_get(s);
9603                 CHECK(params, ENOMEM);
9604
9605                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9606
9607                 selector_params_free(params);
9608                 CHECK(ts->obj, ENODEV);
9609         }
9610
9611         TAILQ_FOREACH(l, &p->learners, node) {
9612                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9613                         p->n_selectors + l->id];
9614                 struct rte_swx_table_learner_params *params;
9615
9616                 /* ts->obj. */
9617                 params = learner_params_get(l);
9618                 CHECK(params, ENOMEM);
9619
9620                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9621                 learner_params_free(params);
9622                 CHECK(ts->obj, ENODEV);
9623
9624                 /* ts->default_action_data. */
9625                 if (l->action_data_size_max) {
9626                         ts->default_action_data = malloc(l->action_data_size_max);
9627                         CHECK(ts->default_action_data, ENOMEM);
9628
9629                         memcpy(ts->default_action_data,
9630                                l->default_action_data,
9631                                l->action_data_size_max);
9632                 }
9633
9634                 /* ts->default_action_id. */
9635                 ts->default_action_id = l->default_action->id;
9636         }
9637
9638         return 0;
9639 }
9640
9641 static void
9642 table_state_build_free(struct rte_swx_pipeline *p)
9643 {
9644         uint32_t i;
9645
9646         if (!p->table_state)
9647                 return;
9648
9649         for (i = 0; i < p->n_tables; i++) {
9650                 struct rte_swx_table_state *ts = &p->table_state[i];
9651                 struct table *table = table_find_by_id(p, i);
9652
9653                 /* ts->obj. */
9654                 if (table->type && ts->obj)
9655                         table->type->ops.free(ts->obj);
9656
9657                 /* ts->default_action_data. */
9658                 free(ts->default_action_data);
9659         }
9660
9661         for (i = 0; i < p->n_selectors; i++) {
9662                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9663
9664                 /* ts->obj. */
9665                 if (ts->obj)
9666                         rte_swx_table_selector_free(ts->obj);
9667         }
9668
9669         for (i = 0; i < p->n_learners; i++) {
9670                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9671
9672                 /* ts->obj. */
9673                 if (ts->obj)
9674                         rte_swx_table_learner_free(ts->obj);
9675
9676                 /* ts->default_action_data. */
9677                 free(ts->default_action_data);
9678         }
9679
9680         free(p->table_state);
9681         p->table_state = NULL;
9682 }
9683
9684 static void
9685 table_state_free(struct rte_swx_pipeline *p)
9686 {
9687         table_state_build_free(p);
9688 }
9689
9690 /*
9691  * Register array.
9692  */
9693 static struct regarray *
9694 regarray_find(struct rte_swx_pipeline *p, const char *name)
9695 {
9696         struct regarray *elem;
9697
9698         TAILQ_FOREACH(elem, &p->regarrays, node)
9699                 if (!strcmp(elem->name, name))
9700                         return elem;
9701
9702         return NULL;
9703 }
9704
9705 static struct regarray *
9706 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9707 {
9708         struct regarray *elem = NULL;
9709
9710         TAILQ_FOREACH(elem, &p->regarrays, node)
9711                 if (elem->id == id)
9712                         return elem;
9713
9714         return NULL;
9715 }
9716
9717 int
9718 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9719                               const char *name,
9720                               uint32_t size,
9721                               uint64_t init_val)
9722 {
9723         struct regarray *r;
9724
9725         CHECK(p, EINVAL);
9726
9727         CHECK_NAME(name, EINVAL);
9728         CHECK(!regarray_find(p, name), EEXIST);
9729
9730         CHECK(size, EINVAL);
9731         size = rte_align32pow2(size);
9732
9733         /* Memory allocation. */
9734         r = calloc(1, sizeof(struct regarray));
9735         CHECK(r, ENOMEM);
9736
9737         /* Node initialization. */
9738         strcpy(r->name, name);
9739         r->init_val = init_val;
9740         r->size = size;
9741         r->id = p->n_regarrays;
9742
9743         /* Node add to tailq. */
9744         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9745         p->n_regarrays++;
9746
9747         return 0;
9748 }
9749
9750 static int
9751 regarray_build(struct rte_swx_pipeline *p)
9752 {
9753         struct regarray *regarray;
9754
9755         if (!p->n_regarrays)
9756                 return 0;
9757
9758         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9759         CHECK(p->regarray_runtime, ENOMEM);
9760
9761         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9762                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9763                 uint32_t i;
9764
9765                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9766                                          RTE_CACHE_LINE_SIZE,
9767                                          p->numa_node);
9768                 CHECK(r->regarray, ENOMEM);
9769
9770                 if (regarray->init_val)
9771                         for (i = 0; i < regarray->size; i++)
9772                                 r->regarray[i] = regarray->init_val;
9773
9774                 r->size_mask = regarray->size - 1;
9775         }
9776
9777         return 0;
9778 }
9779
9780 static void
9781 regarray_build_free(struct rte_swx_pipeline *p)
9782 {
9783         uint32_t i;
9784
9785         if (!p->regarray_runtime)
9786                 return;
9787
9788         for (i = 0; i < p->n_regarrays; i++) {
9789                 struct regarray *regarray = regarray_find_by_id(p, i);
9790                 struct regarray_runtime *r = &p->regarray_runtime[i];
9791
9792                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9793         }
9794
9795         free(p->regarray_runtime);
9796         p->regarray_runtime = NULL;
9797 }
9798
9799 static void
9800 regarray_free(struct rte_swx_pipeline *p)
9801 {
9802         regarray_build_free(p);
9803
9804         for ( ; ; ) {
9805                 struct regarray *elem;
9806
9807                 elem = TAILQ_FIRST(&p->regarrays);
9808                 if (!elem)
9809                         break;
9810
9811                 TAILQ_REMOVE(&p->regarrays, elem, node);
9812                 free(elem);
9813         }
9814 }
9815
9816 /*
9817  * Meter array.
9818  */
9819 static struct meter_profile *
9820 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9821 {
9822         struct meter_profile *elem;
9823
9824         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9825                 if (!strcmp(elem->name, name))
9826                         return elem;
9827
9828         return NULL;
9829 }
9830
9831 static struct metarray *
9832 metarray_find(struct rte_swx_pipeline *p, const char *name)
9833 {
9834         struct metarray *elem;
9835
9836         TAILQ_FOREACH(elem, &p->metarrays, node)
9837                 if (!strcmp(elem->name, name))
9838                         return elem;
9839
9840         return NULL;
9841 }
9842
9843 static struct metarray *
9844 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9845 {
9846         struct metarray *elem = NULL;
9847
9848         TAILQ_FOREACH(elem, &p->metarrays, node)
9849                 if (elem->id == id)
9850                         return elem;
9851
9852         return NULL;
9853 }
9854
9855 int
9856 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9857                                  const char *name,
9858                                  uint32_t size)
9859 {
9860         struct metarray *m;
9861
9862         CHECK(p, EINVAL);
9863
9864         CHECK_NAME(name, EINVAL);
9865         CHECK(!metarray_find(p, name), EEXIST);
9866
9867         CHECK(size, EINVAL);
9868         size = rte_align32pow2(size);
9869
9870         /* Memory allocation. */
9871         m = calloc(1, sizeof(struct metarray));
9872         CHECK(m, ENOMEM);
9873
9874         /* Node initialization. */
9875         strcpy(m->name, name);
9876         m->size = size;
9877         m->id = p->n_metarrays;
9878
9879         /* Node add to tailq. */
9880         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9881         p->n_metarrays++;
9882
9883         return 0;
9884 }
9885
9886 struct meter_profile meter_profile_default = {
9887         .node = {0},
9888         .name = "",
9889         .params = {0},
9890
9891         .profile = {
9892                 .cbs = 10000,
9893                 .pbs = 10000,
9894                 .cir_period = 1,
9895                 .cir_bytes_per_period = 1,
9896                 .pir_period = 1,
9897                 .pir_bytes_per_period = 1,
9898         },
9899
9900         .n_users = 0,
9901 };
9902
9903 static void
9904 meter_init(struct meter *m)
9905 {
9906         memset(m, 0, sizeof(struct meter));
9907         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9908         m->profile = &meter_profile_default;
9909         m->color_mask = RTE_COLOR_GREEN;
9910
9911         meter_profile_default.n_users++;
9912 }
9913
9914 static int
9915 metarray_build(struct rte_swx_pipeline *p)
9916 {
9917         struct metarray *m;
9918
9919         if (!p->n_metarrays)
9920                 return 0;
9921
9922         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9923         CHECK(p->metarray_runtime, ENOMEM);
9924
9925         TAILQ_FOREACH(m, &p->metarrays, node) {
9926                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9927                 uint32_t i;
9928
9929                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9930                                          RTE_CACHE_LINE_SIZE,
9931                                          p->numa_node);
9932                 CHECK(r->metarray, ENOMEM);
9933
9934                 for (i = 0; i < m->size; i++)
9935                         meter_init(&r->metarray[i]);
9936
9937                 r->size_mask = m->size - 1;
9938         }
9939
9940         return 0;
9941 }
9942
9943 static void
9944 metarray_build_free(struct rte_swx_pipeline *p)
9945 {
9946         uint32_t i;
9947
9948         if (!p->metarray_runtime)
9949                 return;
9950
9951         for (i = 0; i < p->n_metarrays; i++) {
9952                 struct metarray *m = metarray_find_by_id(p, i);
9953                 struct metarray_runtime *r = &p->metarray_runtime[i];
9954
9955                 env_free(r->metarray, m->size * sizeof(struct meter));
9956         }
9957
9958         free(p->metarray_runtime);
9959         p->metarray_runtime = NULL;
9960 }
9961
9962 static void
9963 metarray_free(struct rte_swx_pipeline *p)
9964 {
9965         metarray_build_free(p);
9966
9967         /* Meter arrays. */
9968         for ( ; ; ) {
9969                 struct metarray *elem;
9970
9971                 elem = TAILQ_FIRST(&p->metarrays);
9972                 if (!elem)
9973                         break;
9974
9975                 TAILQ_REMOVE(&p->metarrays, elem, node);
9976                 free(elem);
9977         }
9978
9979         /* Meter profiles. */
9980         for ( ; ; ) {
9981                 struct meter_profile *elem;
9982
9983                 elem = TAILQ_FIRST(&p->meter_profiles);
9984                 if (!elem)
9985                         break;
9986
9987                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9988                 free(elem);
9989         }
9990 }
9991
9992 /*
9993  * Pipeline.
9994  */
9995 int
9996 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9997 {
9998         struct rte_swx_pipeline *pipeline;
9999
10000         /* Check input parameters. */
10001         CHECK(p, EINVAL);
10002
10003         /* Memory allocation. */
10004         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
10005         CHECK(pipeline, ENOMEM);
10006
10007         /* Initialization. */
10008         TAILQ_INIT(&pipeline->struct_types);
10009         TAILQ_INIT(&pipeline->port_in_types);
10010         TAILQ_INIT(&pipeline->ports_in);
10011         TAILQ_INIT(&pipeline->port_out_types);
10012         TAILQ_INIT(&pipeline->ports_out);
10013         TAILQ_INIT(&pipeline->extern_types);
10014         TAILQ_INIT(&pipeline->extern_objs);
10015         TAILQ_INIT(&pipeline->extern_funcs);
10016         TAILQ_INIT(&pipeline->headers);
10017         TAILQ_INIT(&pipeline->actions);
10018         TAILQ_INIT(&pipeline->table_types);
10019         TAILQ_INIT(&pipeline->tables);
10020         TAILQ_INIT(&pipeline->selectors);
10021         TAILQ_INIT(&pipeline->learners);
10022         TAILQ_INIT(&pipeline->regarrays);
10023         TAILQ_INIT(&pipeline->meter_profiles);
10024         TAILQ_INIT(&pipeline->metarrays);
10025
10026         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
10027         pipeline->numa_node = numa_node;
10028
10029         *p = pipeline;
10030         return 0;
10031 }
10032
10033 void
10034 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
10035 {
10036         if (!p)
10037                 return;
10038
10039         free(p->instructions);
10040
10041         metarray_free(p);
10042         regarray_free(p);
10043         table_state_free(p);
10044         learner_free(p);
10045         selector_free(p);
10046         table_free(p);
10047         action_free(p);
10048         metadata_free(p);
10049         header_free(p);
10050         extern_func_free(p);
10051         extern_obj_free(p);
10052         port_out_free(p);
10053         port_in_free(p);
10054         struct_free(p);
10055
10056         free(p);
10057 }
10058
10059 int
10060 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
10061                                      const char **instructions,
10062                                      uint32_t n_instructions)
10063 {
10064         int err;
10065         uint32_t i;
10066
10067         err = instruction_config(p, NULL, instructions, n_instructions);
10068         if (err)
10069                 return err;
10070
10071         /* Thread instruction pointer reset. */
10072         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10073                 struct thread *t = &p->threads[i];
10074
10075                 thread_ip_reset(p, t);
10076         }
10077
10078         return 0;
10079 }
10080
10081 int
10082 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10083 {
10084         int status;
10085
10086         CHECK(p, EINVAL);
10087         CHECK(p->build_done == 0, EEXIST);
10088
10089         status = port_in_build(p);
10090         if (status)
10091                 goto error;
10092
10093         status = port_out_build(p);
10094         if (status)
10095                 goto error;
10096
10097         status = struct_build(p);
10098         if (status)
10099                 goto error;
10100
10101         status = extern_obj_build(p);
10102         if (status)
10103                 goto error;
10104
10105         status = extern_func_build(p);
10106         if (status)
10107                 goto error;
10108
10109         status = header_build(p);
10110         if (status)
10111                 goto error;
10112
10113         status = metadata_build(p);
10114         if (status)
10115                 goto error;
10116
10117         status = action_build(p);
10118         if (status)
10119                 goto error;
10120
10121         status = table_build(p);
10122         if (status)
10123                 goto error;
10124
10125         status = selector_build(p);
10126         if (status)
10127                 goto error;
10128
10129         status = learner_build(p);
10130         if (status)
10131                 goto error;
10132
10133         status = table_state_build(p);
10134         if (status)
10135                 goto error;
10136
10137         status = regarray_build(p);
10138         if (status)
10139                 goto error;
10140
10141         status = metarray_build(p);
10142         if (status)
10143                 goto error;
10144
10145         p->build_done = 1;
10146         return 0;
10147
10148 error:
10149         metarray_build_free(p);
10150         regarray_build_free(p);
10151         table_state_build_free(p);
10152         learner_build_free(p);
10153         selector_build_free(p);
10154         table_build_free(p);
10155         action_build_free(p);
10156         metadata_build_free(p);
10157         header_build_free(p);
10158         extern_func_build_free(p);
10159         extern_obj_build_free(p);
10160         port_out_build_free(p);
10161         port_in_build_free(p);
10162         struct_build_free(p);
10163
10164         return status;
10165 }
10166
10167 void
10168 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10169 {
10170         uint32_t i;
10171
10172         for (i = 0; i < n_instructions; i++)
10173                 instr_exec(p);
10174 }
10175
10176 void
10177 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10178 {
10179         uint32_t i;
10180
10181         for (i = 0; i < p->n_ports_out; i++) {
10182                 struct port_out_runtime *port = &p->out[i];
10183
10184                 if (port->flush)
10185                         port->flush(port->obj);
10186         }
10187 }
10188
10189 /*
10190  * Control.
10191  */
10192 int
10193 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10194                               struct rte_swx_ctl_pipeline_info *pipeline)
10195 {
10196         struct action *action;
10197         struct table *table;
10198         uint32_t n_actions = 0, n_tables = 0;
10199
10200         if (!p || !pipeline)
10201                 return -EINVAL;
10202
10203         TAILQ_FOREACH(action, &p->actions, node)
10204                 n_actions++;
10205
10206         TAILQ_FOREACH(table, &p->tables, node)
10207                 n_tables++;
10208
10209         pipeline->n_ports_in = p->n_ports_in;
10210         pipeline->n_ports_out = p->n_ports_out;
10211         pipeline->n_actions = n_actions;
10212         pipeline->n_tables = n_tables;
10213         pipeline->n_selectors = p->n_selectors;
10214         pipeline->n_learners = p->n_learners;
10215         pipeline->n_regarrays = p->n_regarrays;
10216         pipeline->n_metarrays = p->n_metarrays;
10217
10218         return 0;
10219 }
10220
10221 int
10222 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10223 {
10224         if (!p || !numa_node)
10225                 return -EINVAL;
10226
10227         *numa_node = p->numa_node;
10228         return 0;
10229 }
10230
10231 int
10232 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10233                             uint32_t action_id,
10234                             struct rte_swx_ctl_action_info *action)
10235 {
10236         struct action *a = NULL;
10237
10238         if (!p || (action_id >= p->n_actions) || !action)
10239                 return -EINVAL;
10240
10241         a = action_find_by_id(p, action_id);
10242         if (!a)
10243                 return -EINVAL;
10244
10245         strcpy(action->name, a->name);
10246         action->n_args = a->st ? a->st->n_fields : 0;
10247         return 0;
10248 }
10249
10250 int
10251 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10252                                 uint32_t action_id,
10253                                 uint32_t action_arg_id,
10254                                 struct rte_swx_ctl_action_arg_info *action_arg)
10255 {
10256         struct action *a = NULL;
10257         struct field *arg = NULL;
10258
10259         if (!p || (action_id >= p->n_actions) || !action_arg)
10260                 return -EINVAL;
10261
10262         a = action_find_by_id(p, action_id);
10263         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10264                 return -EINVAL;
10265
10266         arg = &a->st->fields[action_arg_id];
10267         strcpy(action_arg->name, arg->name);
10268         action_arg->n_bits = arg->n_bits;
10269         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10270
10271         return 0;
10272 }
10273
10274 int
10275 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10276                            uint32_t table_id,
10277                            struct rte_swx_ctl_table_info *table)
10278 {
10279         struct table *t = NULL;
10280
10281         if (!p || !table)
10282                 return -EINVAL;
10283
10284         t = table_find_by_id(p, table_id);
10285         if (!t)
10286                 return -EINVAL;
10287
10288         strcpy(table->name, t->name);
10289         strcpy(table->args, t->args);
10290         table->n_match_fields = t->n_fields;
10291         table->n_actions = t->n_actions;
10292         table->default_action_is_const = t->default_action_is_const;
10293         table->size = t->size;
10294         return 0;
10295 }
10296
10297 int
10298 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10299         uint32_t table_id,
10300         uint32_t match_field_id,
10301         struct rte_swx_ctl_table_match_field_info *match_field)
10302 {
10303         struct table *t;
10304         struct match_field *f;
10305
10306         if (!p || (table_id >= p->n_tables) || !match_field)
10307                 return -EINVAL;
10308
10309         t = table_find_by_id(p, table_id);
10310         if (!t || (match_field_id >= t->n_fields))
10311                 return -EINVAL;
10312
10313         f = &t->fields[match_field_id];
10314         match_field->match_type = f->match_type;
10315         match_field->is_header = t->header ? 1 : 0;
10316         match_field->n_bits = f->field->n_bits;
10317         match_field->offset = f->field->offset;
10318
10319         return 0;
10320 }
10321
10322 int
10323 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10324         uint32_t table_id,
10325         uint32_t table_action_id,
10326         struct rte_swx_ctl_table_action_info *table_action)
10327 {
10328         struct table *t;
10329
10330         if (!p || (table_id >= p->n_tables) || !table_action)
10331                 return -EINVAL;
10332
10333         t = table_find_by_id(p, table_id);
10334         if (!t || (table_action_id >= t->n_actions))
10335                 return -EINVAL;
10336
10337         table_action->action_id = t->actions[table_action_id]->id;
10338
10339         return 0;
10340 }
10341
10342 int
10343 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10344                           uint32_t table_id,
10345                           struct rte_swx_table_ops *table_ops,
10346                           int *is_stub)
10347 {
10348         struct table *t;
10349
10350         if (!p || (table_id >= p->n_tables))
10351                 return -EINVAL;
10352
10353         t = table_find_by_id(p, table_id);
10354         if (!t)
10355                 return -EINVAL;
10356
10357         if (t->type) {
10358                 if (table_ops)
10359                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10360                 *is_stub = 0;
10361         } else {
10362                 *is_stub = 1;
10363         }
10364
10365         return 0;
10366 }
10367
10368 int
10369 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10370                               uint32_t selector_id,
10371                               struct rte_swx_ctl_selector_info *selector)
10372 {
10373         struct selector *s = NULL;
10374
10375         if (!p || !selector)
10376                 return -EINVAL;
10377
10378         s = selector_find_by_id(p, selector_id);
10379         if (!s)
10380                 return -EINVAL;
10381
10382         strcpy(selector->name, s->name);
10383
10384         selector->n_selector_fields = s->n_selector_fields;
10385         selector->n_groups_max = s->n_groups_max;
10386         selector->n_members_per_group_max = s->n_members_per_group_max;
10387
10388         return 0;
10389 }
10390
10391 int
10392 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10393          uint32_t selector_id,
10394          struct rte_swx_ctl_table_match_field_info *field)
10395 {
10396         struct selector *s;
10397
10398         if (!p || (selector_id >= p->n_selectors) || !field)
10399                 return -EINVAL;
10400
10401         s = selector_find_by_id(p, selector_id);
10402         if (!s)
10403                 return -EINVAL;
10404
10405         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10406         field->is_header = 0;
10407         field->n_bits = s->group_id_field->n_bits;
10408         field->offset = s->group_id_field->offset;
10409
10410         return 0;
10411 }
10412
10413 int
10414 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10415          uint32_t selector_id,
10416          uint32_t selector_field_id,
10417          struct rte_swx_ctl_table_match_field_info *field)
10418 {
10419         struct selector *s;
10420         struct field *f;
10421
10422         if (!p || (selector_id >= p->n_selectors) || !field)
10423                 return -EINVAL;
10424
10425         s = selector_find_by_id(p, selector_id);
10426         if (!s || (selector_field_id >= s->n_selector_fields))
10427                 return -EINVAL;
10428
10429         f = s->selector_fields[selector_field_id];
10430         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10431         field->is_header = s->selector_header ? 1 : 0;
10432         field->n_bits = f->n_bits;
10433         field->offset = f->offset;
10434
10435         return 0;
10436 }
10437
10438 int
10439 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10440          uint32_t selector_id,
10441          struct rte_swx_ctl_table_match_field_info *field)
10442 {
10443         struct selector *s;
10444
10445         if (!p || (selector_id >= p->n_selectors) || !field)
10446                 return -EINVAL;
10447
10448         s = selector_find_by_id(p, selector_id);
10449         if (!s)
10450                 return -EINVAL;
10451
10452         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10453         field->is_header = 0;
10454         field->n_bits = s->member_id_field->n_bits;
10455         field->offset = s->member_id_field->offset;
10456
10457         return 0;
10458 }
10459
10460 int
10461 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10462                              uint32_t learner_id,
10463                              struct rte_swx_ctl_learner_info *learner)
10464 {
10465         struct learner *l = NULL;
10466
10467         if (!p || !learner)
10468                 return -EINVAL;
10469
10470         l = learner_find_by_id(p, learner_id);
10471         if (!l)
10472                 return -EINVAL;
10473
10474         strcpy(learner->name, l->name);
10475
10476         learner->n_match_fields = l->n_fields;
10477         learner->n_actions = l->n_actions;
10478         learner->default_action_is_const = l->default_action_is_const;
10479         learner->size = l->size;
10480
10481         return 0;
10482 }
10483
10484 int
10485 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10486                                          uint32_t learner_id,
10487                                          uint32_t match_field_id,
10488                                          struct rte_swx_ctl_table_match_field_info *match_field)
10489 {
10490         struct learner *l;
10491         struct field *f;
10492
10493         if (!p || (learner_id >= p->n_learners) || !match_field)
10494                 return -EINVAL;
10495
10496         l = learner_find_by_id(p, learner_id);
10497         if (!l || (match_field_id >= l->n_fields))
10498                 return -EINVAL;
10499
10500         f = l->fields[match_field_id];
10501         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10502         match_field->is_header = l->header ? 1 : 0;
10503         match_field->n_bits = f->n_bits;
10504         match_field->offset = f->offset;
10505
10506         return 0;
10507 }
10508
10509 int
10510 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10511                                     uint32_t learner_id,
10512                                     uint32_t learner_action_id,
10513                                     struct rte_swx_ctl_table_action_info *learner_action)
10514 {
10515         struct learner *l;
10516
10517         if (!p || (learner_id >= p->n_learners) || !learner_action)
10518                 return -EINVAL;
10519
10520         l = learner_find_by_id(p, learner_id);
10521         if (!l || (learner_action_id >= l->n_actions))
10522                 return -EINVAL;
10523
10524         learner_action->action_id = l->actions[learner_action_id]->id;
10525
10526         return 0;
10527 }
10528
10529 int
10530 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10531                                  struct rte_swx_table_state **table_state)
10532 {
10533         if (!p || !table_state || !p->build_done)
10534                 return -EINVAL;
10535
10536         *table_state = p->table_state;
10537         return 0;
10538 }
10539
10540 int
10541 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10542                                  struct rte_swx_table_state *table_state)
10543 {
10544         if (!p || !table_state || !p->build_done)
10545                 return -EINVAL;
10546
10547         p->table_state = table_state;
10548         return 0;
10549 }
10550
10551 int
10552 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10553                                         uint32_t port_id,
10554                                         struct rte_swx_port_in_stats *stats)
10555 {
10556         struct port_in *port;
10557
10558         if (!p || !stats)
10559                 return -EINVAL;
10560
10561         port = port_in_find(p, port_id);
10562         if (!port)
10563                 return -EINVAL;
10564
10565         port->type->ops.stats_read(port->obj, stats);
10566         return 0;
10567 }
10568
10569 int
10570 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10571                                          uint32_t port_id,
10572                                          struct rte_swx_port_out_stats *stats)
10573 {
10574         struct port_out *port;
10575
10576         if (!p || !stats)
10577                 return -EINVAL;
10578
10579         port = port_out_find(p, port_id);
10580         if (!port)
10581                 return -EINVAL;
10582
10583         port->type->ops.stats_read(port->obj, stats);
10584         return 0;
10585 }
10586
10587 int
10588 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10589                                       const char *table_name,
10590                                       struct rte_swx_table_stats *stats)
10591 {
10592         struct table *table;
10593         struct table_statistics *table_stats;
10594
10595         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10596                 return -EINVAL;
10597
10598         table = table_find(p, table_name);
10599         if (!table)
10600                 return -EINVAL;
10601
10602         table_stats = &p->table_stats[table->id];
10603
10604         memcpy(stats->n_pkts_action,
10605                table_stats->n_pkts_action,
10606                p->n_actions * sizeof(uint64_t));
10607
10608         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10609         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10610
10611         return 0;
10612 }
10613
10614 int
10615 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10616         const char *selector_name,
10617         struct rte_swx_pipeline_selector_stats *stats)
10618 {
10619         struct selector *s;
10620
10621         if (!p || !selector_name || !selector_name[0] || !stats)
10622                 return -EINVAL;
10623
10624         s = selector_find(p, selector_name);
10625         if (!s)
10626                 return -EINVAL;
10627
10628         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10629
10630         return 0;
10631 }
10632
10633 int
10634 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10635                                         const char *learner_name,
10636                                         struct rte_swx_learner_stats *stats)
10637 {
10638         struct learner *l;
10639         struct learner_statistics *learner_stats;
10640
10641         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10642                 return -EINVAL;
10643
10644         l = learner_find(p, learner_name);
10645         if (!l)
10646                 return -EINVAL;
10647
10648         learner_stats = &p->learner_stats[l->id];
10649
10650         memcpy(stats->n_pkts_action,
10651                learner_stats->n_pkts_action,
10652                p->n_actions * sizeof(uint64_t));
10653
10654         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10655         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10656
10657         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10658         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10659
10660         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10661
10662         return 0;
10663 }
10664
10665 int
10666 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10667                               uint32_t regarray_id,
10668                               struct rte_swx_ctl_regarray_info *regarray)
10669 {
10670         struct regarray *r;
10671
10672         if (!p || !regarray)
10673                 return -EINVAL;
10674
10675         r = regarray_find_by_id(p, regarray_id);
10676         if (!r)
10677                 return -EINVAL;
10678
10679         strcpy(regarray->name, r->name);
10680         regarray->size = r->size;
10681         return 0;
10682 }
10683
10684 int
10685 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10686                                    const char *regarray_name,
10687                                    uint32_t regarray_index,
10688                                    uint64_t *value)
10689 {
10690         struct regarray *regarray;
10691         struct regarray_runtime *r;
10692
10693         if (!p || !regarray_name || !value)
10694                 return -EINVAL;
10695
10696         regarray = regarray_find(p, regarray_name);
10697         if (!regarray || (regarray_index >= regarray->size))
10698                 return -EINVAL;
10699
10700         r = &p->regarray_runtime[regarray->id];
10701         *value = r->regarray[regarray_index];
10702         return 0;
10703 }
10704
10705 int
10706 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10707                                    const char *regarray_name,
10708                                    uint32_t regarray_index,
10709                                    uint64_t value)
10710 {
10711         struct regarray *regarray;
10712         struct regarray_runtime *r;
10713
10714         if (!p || !regarray_name)
10715                 return -EINVAL;
10716
10717         regarray = regarray_find(p, regarray_name);
10718         if (!regarray || (regarray_index >= regarray->size))
10719                 return -EINVAL;
10720
10721         r = &p->regarray_runtime[regarray->id];
10722         r->regarray[regarray_index] = value;
10723         return 0;
10724 }
10725
10726 int
10727 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10728                               uint32_t metarray_id,
10729                               struct rte_swx_ctl_metarray_info *metarray)
10730 {
10731         struct metarray *m;
10732
10733         if (!p || !metarray)
10734                 return -EINVAL;
10735
10736         m = metarray_find_by_id(p, metarray_id);
10737         if (!m)
10738                 return -EINVAL;
10739
10740         strcpy(metarray->name, m->name);
10741         metarray->size = m->size;
10742         return 0;
10743 }
10744
10745 int
10746 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10747                               const char *name,
10748                               struct rte_meter_trtcm_params *params)
10749 {
10750         struct meter_profile *mp;
10751         int status;
10752
10753         CHECK(p, EINVAL);
10754         CHECK_NAME(name, EINVAL);
10755         CHECK(params, EINVAL);
10756         CHECK(!meter_profile_find(p, name), EEXIST);
10757
10758         /* Node allocation. */
10759         mp = calloc(1, sizeof(struct meter_profile));
10760         CHECK(mp, ENOMEM);
10761
10762         /* Node initialization. */
10763         strcpy(mp->name, name);
10764         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10765         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10766         if (status) {
10767                 free(mp);
10768                 CHECK(0, EINVAL);
10769         }
10770
10771         /* Node add to tailq. */
10772         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10773
10774         return 0;
10775 }
10776
10777 int
10778 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10779                                  const char *name)
10780 {
10781         struct meter_profile *mp;
10782
10783         CHECK(p, EINVAL);
10784         CHECK_NAME(name, EINVAL);
10785
10786         mp = meter_profile_find(p, name);
10787         CHECK(mp, EINVAL);
10788         CHECK(!mp->n_users, EBUSY);
10789
10790         /* Remove node from tailq. */
10791         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10792         free(mp);
10793
10794         return 0;
10795 }
10796
10797 int
10798 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10799                         const char *metarray_name,
10800                         uint32_t metarray_index)
10801 {
10802         struct meter_profile *mp_old;
10803         struct metarray *metarray;
10804         struct metarray_runtime *metarray_runtime;
10805         struct meter *m;
10806
10807         CHECK(p, EINVAL);
10808         CHECK_NAME(metarray_name, EINVAL);
10809
10810         metarray = metarray_find(p, metarray_name);
10811         CHECK(metarray, EINVAL);
10812         CHECK(metarray_index < metarray->size, EINVAL);
10813
10814         metarray_runtime = &p->metarray_runtime[metarray->id];
10815         m = &metarray_runtime->metarray[metarray_index];
10816         mp_old = m->profile;
10817
10818         meter_init(m);
10819
10820         mp_old->n_users--;
10821
10822         return 0;
10823 }
10824
10825 int
10826 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10827                       const char *metarray_name,
10828                       uint32_t metarray_index,
10829                       const char *profile_name)
10830 {
10831         struct meter_profile *mp, *mp_old;
10832         struct metarray *metarray;
10833         struct metarray_runtime *metarray_runtime;
10834         struct meter *m;
10835
10836         CHECK(p, EINVAL);
10837         CHECK_NAME(metarray_name, EINVAL);
10838
10839         metarray = metarray_find(p, metarray_name);
10840         CHECK(metarray, EINVAL);
10841         CHECK(metarray_index < metarray->size, EINVAL);
10842
10843         mp = meter_profile_find(p, profile_name);
10844         CHECK(mp, EINVAL);
10845
10846         metarray_runtime = &p->metarray_runtime[metarray->id];
10847         m = &metarray_runtime->metarray[metarray_index];
10848         mp_old = m->profile;
10849
10850         memset(m, 0, sizeof(struct meter));
10851         rte_meter_trtcm_config(&m->m, &mp->profile);
10852         m->profile = mp;
10853         m->color_mask = RTE_COLORS;
10854
10855         mp->n_users++;
10856         mp_old->n_users--;
10857
10858         return 0;
10859 }
10860
10861 int
10862 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10863                              const char *metarray_name,
10864                              uint32_t metarray_index,
10865                              struct rte_swx_ctl_meter_stats *stats)
10866 {
10867         struct metarray *metarray;
10868         struct metarray_runtime *metarray_runtime;
10869         struct meter *m;
10870
10871         CHECK(p, EINVAL);
10872         CHECK_NAME(metarray_name, EINVAL);
10873
10874         metarray = metarray_find(p, metarray_name);
10875         CHECK(metarray, EINVAL);
10876         CHECK(metarray_index < metarray->size, EINVAL);
10877
10878         CHECK(stats, EINVAL);
10879
10880         metarray_runtime = &p->metarray_runtime[metarray->id];
10881         m = &metarray_runtime->metarray[metarray_index];
10882
10883         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10884         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10885
10886         return 0;
10887 }