lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t selector_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107         struct selector_runtime *selector = &t->selectors[selector_id];
2108         struct selector_statistics *stats = &p->selector_stats[selector_id];
2109         uint64_t n_pkts = stats->n_pkts;
2110         int done;
2111
2112         /* Table. */
2113         done = rte_swx_table_selector_select(ts->obj,
2114                            selector->mailbox,
2115                            selector->group_id_buffer,
2116                            selector->selector_buffer,
2117                            selector->member_id_buffer);
2118         if (!done) {
2119                 /* Thread. */
2120                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2121                       p->thread_id,
2122                       selector_id);
2123
2124                 thread_yield(p);
2125                 return;
2126         }
2127
2128
2129         TRACE("[Thread %2u] selector %u\n",
2130               p->thread_id,
2131               selector_id);
2132
2133         stats->n_pkts = n_pkts + 1;
2134
2135         /* Thread. */
2136         thread_ip_inc(p);
2137 }
2138
2139 static inline void
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t learner_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146                 p->n_selectors + learner_id];
2147         struct learner_runtime *l = &t->learners[learner_id];
2148         struct learner_statistics *stats = &p->learner_stats[learner_id];
2149         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150         uint8_t *action_data;
2151         int done, hit;
2152
2153         /* Table. */
2154         time = rte_get_tsc_cycles();
2155
2156         done = rte_swx_table_learner_lookup(ts->obj,
2157                                             l->mailbox,
2158                                             time,
2159                                             l->key,
2160                                             &action_id,
2161                                             &action_data,
2162                                             &hit);
2163         if (!done) {
2164                 /* Thread. */
2165                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2166                       p->thread_id,
2167                       learner_id);
2168
2169                 thread_yield(p);
2170                 return;
2171         }
2172
2173         action_id = hit ? action_id : ts->default_action_id;
2174         action_data = hit ? action_data : ts->default_action_data;
2175         n_pkts_hit = stats->n_pkts_hit[hit];
2176         n_pkts_action = stats->n_pkts_action[action_id];
2177
2178         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2179               p->thread_id,
2180               learner_id,
2181               hit ? "hit" : "miss",
2182               (uint32_t)action_id);
2183
2184         t->action_id = action_id;
2185         t->structs[0] = action_data;
2186         t->hit = hit;
2187         t->learner_id = learner_id;
2188         t->time = time;
2189         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2191
2192         /* Thread. */
2193         thread_ip_action_call(p, t, action_id);
2194 }
2195
2196 /*
2197  * learn.
2198  */
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2201
2202 static int
2203 action_has_nbo_args(struct action *a);
2204
2205 static int
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207                       struct action *action,
2208                       char **tokens,
2209                       int n_tokens,
2210                       struct instruction *instr,
2211                       struct instruction_data *data __rte_unused)
2212 {
2213         struct action *a;
2214
2215         CHECK(action, EINVAL);
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         a = action_find(p, tokens[1]);
2219         CHECK(a, EINVAL);
2220         CHECK(!action_has_nbo_args(a), EINVAL);
2221
2222         instr->type = INSTR_LEARNER_LEARN;
2223         instr->learn.action_id = a->id;
2224
2225         return 0;
2226 }
2227
2228 static inline void
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_learn_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_inc(p);
2238 }
2239
2240 /*
2241  * forget.
2242  */
2243 static int
2244 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2245                        struct action *action,
2246                        char **tokens __rte_unused,
2247                        int n_tokens,
2248                        struct instruction *instr,
2249                        struct instruction_data *data __rte_unused)
2250 {
2251         CHECK(action, EINVAL);
2252         CHECK(n_tokens == 1, EINVAL);
2253
2254         instr->type = INSTR_LEARNER_FORGET;
2255
2256         return 0;
2257 }
2258
2259 static inline void
2260 instr_forget_exec(struct rte_swx_pipeline *p)
2261 {
2262         struct thread *t = &p->threads[p->thread_id];
2263         struct instruction *ip = t->ip;
2264
2265         __instr_forget_exec(p, t, ip);
2266
2267         /* Thread. */
2268         thread_ip_inc(p);
2269 }
2270
2271 /*
2272  * extern.
2273  */
2274 static int
2275 instr_extern_translate(struct rte_swx_pipeline *p,
2276                        struct action *action __rte_unused,
2277                        char **tokens,
2278                        int n_tokens,
2279                        struct instruction *instr,
2280                        struct instruction_data *data __rte_unused)
2281 {
2282         char *token = tokens[1];
2283
2284         CHECK(n_tokens == 2, EINVAL);
2285
2286         if (token[0] == 'e') {
2287                 struct extern_obj *obj;
2288                 struct extern_type_member_func *func;
2289
2290                 func = extern_obj_member_func_parse(p, token, &obj);
2291                 CHECK(func, EINVAL);
2292
2293                 instr->type = INSTR_EXTERN_OBJ;
2294                 instr->ext_obj.ext_obj_id = obj->id;
2295                 instr->ext_obj.func_id = func->id;
2296
2297                 return 0;
2298         }
2299
2300         if (token[0] == 'f') {
2301                 struct extern_func *func;
2302
2303                 func = extern_func_parse(p, token);
2304                 CHECK(func, EINVAL);
2305
2306                 instr->type = INSTR_EXTERN_FUNC;
2307                 instr->ext_func.ext_func_id = func->id;
2308
2309                 return 0;
2310         }
2311
2312         CHECK(0, EINVAL);
2313 }
2314
2315 static inline void
2316 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2317 {
2318         struct thread *t = &p->threads[p->thread_id];
2319         struct instruction *ip = t->ip;
2320         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2321         uint32_t func_id = ip->ext_obj.func_id;
2322         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2323         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2324
2325         TRACE("[Thread %2u] extern obj %u member func %u\n",
2326               p->thread_id,
2327               obj_id,
2328               func_id);
2329
2330         /* Extern object member function execute. */
2331         uint32_t done = func(obj->obj, obj->mailbox);
2332
2333         /* Thread. */
2334         thread_ip_inc_cond(t, done);
2335         thread_yield_cond(p, done ^ 1);
2336 }
2337
2338 static inline void
2339 instr_extern_func_exec(struct rte_swx_pipeline *p)
2340 {
2341         struct thread *t = &p->threads[p->thread_id];
2342         struct instruction *ip = t->ip;
2343         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2344         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2345         rte_swx_extern_func_t func = ext_func->func;
2346
2347         TRACE("[Thread %2u] extern func %u\n",
2348               p->thread_id,
2349               ext_func_id);
2350
2351         /* Extern function execute. */
2352         uint32_t done = func(ext_func->mailbox);
2353
2354         /* Thread. */
2355         thread_ip_inc_cond(t, done);
2356         thread_yield_cond(p, done ^ 1);
2357 }
2358
2359 /*
2360  * mov.
2361  */
2362 static int
2363 instr_mov_translate(struct rte_swx_pipeline *p,
2364                     struct action *action,
2365                     char **tokens,
2366                     int n_tokens,
2367                     struct instruction *instr,
2368                     struct instruction_data *data __rte_unused)
2369 {
2370         char *dst = tokens[1], *src = tokens[2];
2371         struct field *fdst, *fsrc;
2372         uint64_t src_val;
2373         uint32_t dst_struct_id = 0, src_struct_id = 0;
2374
2375         CHECK(n_tokens == 3, EINVAL);
2376
2377         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2378         CHECK(fdst, EINVAL);
2379         CHECK(!fdst->var_size, EINVAL);
2380
2381         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2382         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2383         if (fsrc) {
2384                 CHECK(!fsrc->var_size, EINVAL);
2385
2386                 instr->type = INSTR_MOV;
2387                 if (dst[0] != 'h' && src[0] == 'h')
2388                         instr->type = INSTR_MOV_MH;
2389                 if (dst[0] == 'h' && src[0] != 'h')
2390                         instr->type = INSTR_MOV_HM;
2391                 if (dst[0] == 'h' && src[0] == 'h')
2392                         instr->type = INSTR_MOV_HH;
2393
2394                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2395                 instr->mov.dst.n_bits = fdst->n_bits;
2396                 instr->mov.dst.offset = fdst->offset / 8;
2397                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2398                 instr->mov.src.n_bits = fsrc->n_bits;
2399                 instr->mov.src.offset = fsrc->offset / 8;
2400                 return 0;
2401         }
2402
2403         /* MOV_I. */
2404         src_val = strtoull(src, &src, 0);
2405         CHECK(!src[0], EINVAL);
2406
2407         if (dst[0] == 'h')
2408                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2409
2410         instr->type = INSTR_MOV_I;
2411         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2412         instr->mov.dst.n_bits = fdst->n_bits;
2413         instr->mov.dst.offset = fdst->offset / 8;
2414         instr->mov.src_val = src_val;
2415         return 0;
2416 }
2417
2418 static inline void
2419 instr_mov_exec(struct rte_swx_pipeline *p)
2420 {
2421         struct thread *t = &p->threads[p->thread_id];
2422         struct instruction *ip = t->ip;
2423
2424         TRACE("[Thread %2u] mov\n",
2425               p->thread_id);
2426
2427         MOV(t, ip);
2428
2429         /* Thread. */
2430         thread_ip_inc(p);
2431 }
2432
2433 static inline void
2434 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2435 {
2436         struct thread *t = &p->threads[p->thread_id];
2437         struct instruction *ip = t->ip;
2438
2439         TRACE("[Thread %2u] mov (mh)\n",
2440               p->thread_id);
2441
2442         MOV_MH(t, ip);
2443
2444         /* Thread. */
2445         thread_ip_inc(p);
2446 }
2447
2448 static inline void
2449 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2450 {
2451         struct thread *t = &p->threads[p->thread_id];
2452         struct instruction *ip = t->ip;
2453
2454         TRACE("[Thread %2u] mov (hm)\n",
2455               p->thread_id);
2456
2457         MOV_HM(t, ip);
2458
2459         /* Thread. */
2460         thread_ip_inc(p);
2461 }
2462
2463 static inline void
2464 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2465 {
2466         struct thread *t = &p->threads[p->thread_id];
2467         struct instruction *ip = t->ip;
2468
2469         TRACE("[Thread %2u] mov (hh)\n",
2470               p->thread_id);
2471
2472         MOV_HH(t, ip);
2473
2474         /* Thread. */
2475         thread_ip_inc(p);
2476 }
2477
2478 static inline void
2479 instr_mov_i_exec(struct rte_swx_pipeline *p)
2480 {
2481         struct thread *t = &p->threads[p->thread_id];
2482         struct instruction *ip = t->ip;
2483
2484         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2485               p->thread_id,
2486               ip->mov.src_val);
2487
2488         MOV_I(t, ip);
2489
2490         /* Thread. */
2491         thread_ip_inc(p);
2492 }
2493
2494 /*
2495  * dma.
2496  */
2497 static inline void
2498 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2499
2500 static inline void
2501 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2502 {
2503         struct thread *t = &p->threads[p->thread_id];
2504         struct instruction *ip = t->ip;
2505         uint8_t *action_data = t->structs[0];
2506         uint64_t valid_headers = t->valid_headers;
2507         uint32_t i;
2508
2509         for (i = 0; i < n_dma; i++) {
2510                 uint32_t header_id = ip->dma.dst.header_id[i];
2511                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2512                 uint32_t offset = ip->dma.src.offset[i];
2513                 uint32_t n_bytes = ip->dma.n_bytes[i];
2514
2515                 struct header_runtime *h = &t->headers[header_id];
2516                 uint8_t *h_ptr0 = h->ptr0;
2517                 uint8_t *h_ptr = t->structs[struct_id];
2518
2519                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2520                         h_ptr : h_ptr0;
2521                 void *src = &action_data[offset];
2522
2523                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2524
2525                 /* Headers. */
2526                 memcpy(dst, src, n_bytes);
2527                 t->structs[struct_id] = dst;
2528                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2529         }
2530
2531         t->valid_headers = valid_headers;
2532 }
2533
2534 static inline void
2535 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2536 {
2537         __instr_dma_ht_exec(p, 1);
2538
2539         /* Thread. */
2540         thread_ip_inc(p);
2541 }
2542
2543 static inline void
2544 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2545 {
2546         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2547               p->thread_id);
2548
2549         __instr_dma_ht_exec(p, 2);
2550
2551         /* Thread. */
2552         thread_ip_inc(p);
2553 }
2554
2555 static inline void
2556 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2557 {
2558         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2559               p->thread_id);
2560
2561         __instr_dma_ht_exec(p, 3);
2562
2563         /* Thread. */
2564         thread_ip_inc(p);
2565 }
2566
2567 static inline void
2568 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2569 {
2570         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2571               p->thread_id);
2572
2573         __instr_dma_ht_exec(p, 4);
2574
2575         /* Thread. */
2576         thread_ip_inc(p);
2577 }
2578
2579 static inline void
2580 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2581 {
2582         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2583               p->thread_id);
2584
2585         __instr_dma_ht_exec(p, 5);
2586
2587         /* Thread. */
2588         thread_ip_inc(p);
2589 }
2590
2591 static inline void
2592 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2593 {
2594         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2595               p->thread_id);
2596
2597         __instr_dma_ht_exec(p, 6);
2598
2599         /* Thread. */
2600         thread_ip_inc(p);
2601 }
2602
2603 static inline void
2604 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2605 {
2606         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2607               p->thread_id);
2608
2609         __instr_dma_ht_exec(p, 7);
2610
2611         /* Thread. */
2612         thread_ip_inc(p);
2613 }
2614
2615 static inline void
2616 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2617 {
2618         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2619               p->thread_id);
2620
2621         __instr_dma_ht_exec(p, 8);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 /*
2628  * alu.
2629  */
2630 static int
2631 instr_alu_add_translate(struct rte_swx_pipeline *p,
2632                         struct action *action,
2633                         char **tokens,
2634                         int n_tokens,
2635                         struct instruction *instr,
2636                         struct instruction_data *data __rte_unused)
2637 {
2638         char *dst = tokens[1], *src = tokens[2];
2639         struct field *fdst, *fsrc;
2640         uint64_t src_val;
2641         uint32_t dst_struct_id = 0, src_struct_id = 0;
2642
2643         CHECK(n_tokens == 3, EINVAL);
2644
2645         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2646         CHECK(fdst, EINVAL);
2647         CHECK(!fdst->var_size, EINVAL);
2648
2649         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2650         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2651         if (fsrc) {
2652                 CHECK(!fsrc->var_size, EINVAL);
2653
2654                 instr->type = INSTR_ALU_ADD;
2655                 if (dst[0] == 'h' && src[0] != 'h')
2656                         instr->type = INSTR_ALU_ADD_HM;
2657                 if (dst[0] != 'h' && src[0] == 'h')
2658                         instr->type = INSTR_ALU_ADD_MH;
2659                 if (dst[0] == 'h' && src[0] == 'h')
2660                         instr->type = INSTR_ALU_ADD_HH;
2661
2662                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2663                 instr->alu.dst.n_bits = fdst->n_bits;
2664                 instr->alu.dst.offset = fdst->offset / 8;
2665                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2666                 instr->alu.src.n_bits = fsrc->n_bits;
2667                 instr->alu.src.offset = fsrc->offset / 8;
2668                 return 0;
2669         }
2670
2671         /* ADD_MI, ADD_HI. */
2672         src_val = strtoull(src, &src, 0);
2673         CHECK(!src[0], EINVAL);
2674
2675         instr->type = INSTR_ALU_ADD_MI;
2676         if (dst[0] == 'h')
2677                 instr->type = INSTR_ALU_ADD_HI;
2678
2679         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2680         instr->alu.dst.n_bits = fdst->n_bits;
2681         instr->alu.dst.offset = fdst->offset / 8;
2682         instr->alu.src_val = src_val;
2683         return 0;
2684 }
2685
2686 static int
2687 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2688                         struct action *action,
2689                         char **tokens,
2690                         int n_tokens,
2691                         struct instruction *instr,
2692                         struct instruction_data *data __rte_unused)
2693 {
2694         char *dst = tokens[1], *src = tokens[2];
2695         struct field *fdst, *fsrc;
2696         uint64_t src_val;
2697         uint32_t dst_struct_id = 0, src_struct_id = 0;
2698
2699         CHECK(n_tokens == 3, EINVAL);
2700
2701         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2702         CHECK(fdst, EINVAL);
2703         CHECK(!fdst->var_size, EINVAL);
2704
2705         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2706         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2707         if (fsrc) {
2708                 CHECK(!fsrc->var_size, EINVAL);
2709
2710                 instr->type = INSTR_ALU_SUB;
2711                 if (dst[0] == 'h' && src[0] != 'h')
2712                         instr->type = INSTR_ALU_SUB_HM;
2713                 if (dst[0] != 'h' && src[0] == 'h')
2714                         instr->type = INSTR_ALU_SUB_MH;
2715                 if (dst[0] == 'h' && src[0] == 'h')
2716                         instr->type = INSTR_ALU_SUB_HH;
2717
2718                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2719                 instr->alu.dst.n_bits = fdst->n_bits;
2720                 instr->alu.dst.offset = fdst->offset / 8;
2721                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2722                 instr->alu.src.n_bits = fsrc->n_bits;
2723                 instr->alu.src.offset = fsrc->offset / 8;
2724                 return 0;
2725         }
2726
2727         /* SUB_MI, SUB_HI. */
2728         src_val = strtoull(src, &src, 0);
2729         CHECK(!src[0], EINVAL);
2730
2731         instr->type = INSTR_ALU_SUB_MI;
2732         if (dst[0] == 'h')
2733                 instr->type = INSTR_ALU_SUB_HI;
2734
2735         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2736         instr->alu.dst.n_bits = fdst->n_bits;
2737         instr->alu.dst.offset = fdst->offset / 8;
2738         instr->alu.src_val = src_val;
2739         return 0;
2740 }
2741
2742 static int
2743 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2744                           struct action *action __rte_unused,
2745                           char **tokens,
2746                           int n_tokens,
2747                           struct instruction *instr,
2748                           struct instruction_data *data __rte_unused)
2749 {
2750         char *dst = tokens[1], *src = tokens[2];
2751         struct header *hdst, *hsrc;
2752         struct field *fdst, *fsrc;
2753
2754         CHECK(n_tokens == 3, EINVAL);
2755
2756         fdst = header_field_parse(p, dst, &hdst);
2757         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2758         CHECK(!fdst->var_size, EINVAL);
2759
2760         /* CKADD_FIELD. */
2761         fsrc = header_field_parse(p, src, &hsrc);
2762         if (fsrc) {
2763                 CHECK(!fsrc->var_size, EINVAL);
2764
2765                 instr->type = INSTR_ALU_CKADD_FIELD;
2766                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2767                 instr->alu.dst.n_bits = fdst->n_bits;
2768                 instr->alu.dst.offset = fdst->offset / 8;
2769                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2770                 instr->alu.src.n_bits = fsrc->n_bits;
2771                 instr->alu.src.offset = fsrc->offset / 8;
2772                 return 0;
2773         }
2774
2775         /* CKADD_STRUCT, CKADD_STRUCT20. */
2776         hsrc = header_parse(p, src);
2777         CHECK(hsrc, EINVAL);
2778         CHECK(!hsrc->st->var_size, EINVAL);
2779
2780         instr->type = INSTR_ALU_CKADD_STRUCT;
2781         if ((hsrc->st->n_bits / 8) == 20)
2782                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2783
2784         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2785         instr->alu.dst.n_bits = fdst->n_bits;
2786         instr->alu.dst.offset = fdst->offset / 8;
2787         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2788         instr->alu.src.n_bits = hsrc->st->n_bits;
2789         instr->alu.src.offset = 0; /* Unused. */
2790         return 0;
2791 }
2792
2793 static int
2794 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2795                           struct action *action __rte_unused,
2796                           char **tokens,
2797                           int n_tokens,
2798                           struct instruction *instr,
2799                           struct instruction_data *data __rte_unused)
2800 {
2801         char *dst = tokens[1], *src = tokens[2];
2802         struct header *hdst, *hsrc;
2803         struct field *fdst, *fsrc;
2804
2805         CHECK(n_tokens == 3, EINVAL);
2806
2807         fdst = header_field_parse(p, dst, &hdst);
2808         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2809         CHECK(!fdst->var_size, EINVAL);
2810
2811         fsrc = header_field_parse(p, src, &hsrc);
2812         CHECK(fsrc, EINVAL);
2813         CHECK(!fsrc->var_size, EINVAL);
2814
2815         instr->type = INSTR_ALU_CKSUB_FIELD;
2816         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2817         instr->alu.dst.n_bits = fdst->n_bits;
2818         instr->alu.dst.offset = fdst->offset / 8;
2819         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2820         instr->alu.src.n_bits = fsrc->n_bits;
2821         instr->alu.src.offset = fsrc->offset / 8;
2822         return 0;
2823 }
2824
2825 static int
2826 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2827                         struct action *action,
2828                         char **tokens,
2829                         int n_tokens,
2830                         struct instruction *instr,
2831                         struct instruction_data *data __rte_unused)
2832 {
2833         char *dst = tokens[1], *src = tokens[2];
2834         struct field *fdst, *fsrc;
2835         uint64_t src_val;
2836         uint32_t dst_struct_id = 0, src_struct_id = 0;
2837
2838         CHECK(n_tokens == 3, EINVAL);
2839
2840         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2841         CHECK(fdst, EINVAL);
2842         CHECK(!fdst->var_size, EINVAL);
2843
2844         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2845         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2846         if (fsrc) {
2847                 CHECK(!fsrc->var_size, EINVAL);
2848
2849                 instr->type = INSTR_ALU_SHL;
2850                 if (dst[0] == 'h' && src[0] != 'h')
2851                         instr->type = INSTR_ALU_SHL_HM;
2852                 if (dst[0] != 'h' && src[0] == 'h')
2853                         instr->type = INSTR_ALU_SHL_MH;
2854                 if (dst[0] == 'h' && src[0] == 'h')
2855                         instr->type = INSTR_ALU_SHL_HH;
2856
2857                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2858                 instr->alu.dst.n_bits = fdst->n_bits;
2859                 instr->alu.dst.offset = fdst->offset / 8;
2860                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2861                 instr->alu.src.n_bits = fsrc->n_bits;
2862                 instr->alu.src.offset = fsrc->offset / 8;
2863                 return 0;
2864         }
2865
2866         /* SHL_MI, SHL_HI. */
2867         src_val = strtoull(src, &src, 0);
2868         CHECK(!src[0], EINVAL);
2869
2870         instr->type = INSTR_ALU_SHL_MI;
2871         if (dst[0] == 'h')
2872                 instr->type = INSTR_ALU_SHL_HI;
2873
2874         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2875         instr->alu.dst.n_bits = fdst->n_bits;
2876         instr->alu.dst.offset = fdst->offset / 8;
2877         instr->alu.src_val = src_val;
2878         return 0;
2879 }
2880
2881 static int
2882 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2883                         struct action *action,
2884                         char **tokens,
2885                         int n_tokens,
2886                         struct instruction *instr,
2887                         struct instruction_data *data __rte_unused)
2888 {
2889         char *dst = tokens[1], *src = tokens[2];
2890         struct field *fdst, *fsrc;
2891         uint64_t src_val;
2892         uint32_t dst_struct_id = 0, src_struct_id = 0;
2893
2894         CHECK(n_tokens == 3, EINVAL);
2895
2896         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2897         CHECK(fdst, EINVAL);
2898         CHECK(!fdst->var_size, EINVAL);
2899
2900         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2901         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2902         if (fsrc) {
2903                 CHECK(!fsrc->var_size, EINVAL);
2904
2905                 instr->type = INSTR_ALU_SHR;
2906                 if (dst[0] == 'h' && src[0] != 'h')
2907                         instr->type = INSTR_ALU_SHR_HM;
2908                 if (dst[0] != 'h' && src[0] == 'h')
2909                         instr->type = INSTR_ALU_SHR_MH;
2910                 if (dst[0] == 'h' && src[0] == 'h')
2911                         instr->type = INSTR_ALU_SHR_HH;
2912
2913                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2914                 instr->alu.dst.n_bits = fdst->n_bits;
2915                 instr->alu.dst.offset = fdst->offset / 8;
2916                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2917                 instr->alu.src.n_bits = fsrc->n_bits;
2918                 instr->alu.src.offset = fsrc->offset / 8;
2919                 return 0;
2920         }
2921
2922         /* SHR_MI, SHR_HI. */
2923         src_val = strtoull(src, &src, 0);
2924         CHECK(!src[0], EINVAL);
2925
2926         instr->type = INSTR_ALU_SHR_MI;
2927         if (dst[0] == 'h')
2928                 instr->type = INSTR_ALU_SHR_HI;
2929
2930         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2931         instr->alu.dst.n_bits = fdst->n_bits;
2932         instr->alu.dst.offset = fdst->offset / 8;
2933         instr->alu.src_val = src_val;
2934         return 0;
2935 }
2936
2937 static int
2938 instr_alu_and_translate(struct rte_swx_pipeline *p,
2939                         struct action *action,
2940                         char **tokens,
2941                         int n_tokens,
2942                         struct instruction *instr,
2943                         struct instruction_data *data __rte_unused)
2944 {
2945         char *dst = tokens[1], *src = tokens[2];
2946         struct field *fdst, *fsrc;
2947         uint64_t src_val;
2948         uint32_t dst_struct_id = 0, src_struct_id = 0;
2949
2950         CHECK(n_tokens == 3, EINVAL);
2951
2952         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2953         CHECK(fdst, EINVAL);
2954         CHECK(!fdst->var_size, EINVAL);
2955
2956         /* AND, AND_MH, AND_HM, AND_HH. */
2957         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2958         if (fsrc) {
2959                 CHECK(!fsrc->var_size, EINVAL);
2960
2961                 instr->type = INSTR_ALU_AND;
2962                 if (dst[0] != 'h' && src[0] == 'h')
2963                         instr->type = INSTR_ALU_AND_MH;
2964                 if (dst[0] == 'h' && src[0] != 'h')
2965                         instr->type = INSTR_ALU_AND_HM;
2966                 if (dst[0] == 'h' && src[0] == 'h')
2967                         instr->type = INSTR_ALU_AND_HH;
2968
2969                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2970                 instr->alu.dst.n_bits = fdst->n_bits;
2971                 instr->alu.dst.offset = fdst->offset / 8;
2972                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2973                 instr->alu.src.n_bits = fsrc->n_bits;
2974                 instr->alu.src.offset = fsrc->offset / 8;
2975                 return 0;
2976         }
2977
2978         /* AND_I. */
2979         src_val = strtoull(src, &src, 0);
2980         CHECK(!src[0], EINVAL);
2981
2982         if (dst[0] == 'h')
2983                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2984
2985         instr->type = INSTR_ALU_AND_I;
2986         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2987         instr->alu.dst.n_bits = fdst->n_bits;
2988         instr->alu.dst.offset = fdst->offset / 8;
2989         instr->alu.src_val = src_val;
2990         return 0;
2991 }
2992
2993 static int
2994 instr_alu_or_translate(struct rte_swx_pipeline *p,
2995                        struct action *action,
2996                        char **tokens,
2997                        int n_tokens,
2998                        struct instruction *instr,
2999                        struct instruction_data *data __rte_unused)
3000 {
3001         char *dst = tokens[1], *src = tokens[2];
3002         struct field *fdst, *fsrc;
3003         uint64_t src_val;
3004         uint32_t dst_struct_id = 0, src_struct_id = 0;
3005
3006         CHECK(n_tokens == 3, EINVAL);
3007
3008         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3009         CHECK(fdst, EINVAL);
3010         CHECK(!fdst->var_size, EINVAL);
3011
3012         /* OR, OR_MH, OR_HM, OR_HH. */
3013         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3014         if (fsrc) {
3015                 CHECK(!fsrc->var_size, EINVAL);
3016
3017                 instr->type = INSTR_ALU_OR;
3018                 if (dst[0] != 'h' && src[0] == 'h')
3019                         instr->type = INSTR_ALU_OR_MH;
3020                 if (dst[0] == 'h' && src[0] != 'h')
3021                         instr->type = INSTR_ALU_OR_HM;
3022                 if (dst[0] == 'h' && src[0] == 'h')
3023                         instr->type = INSTR_ALU_OR_HH;
3024
3025                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3026                 instr->alu.dst.n_bits = fdst->n_bits;
3027                 instr->alu.dst.offset = fdst->offset / 8;
3028                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3029                 instr->alu.src.n_bits = fsrc->n_bits;
3030                 instr->alu.src.offset = fsrc->offset / 8;
3031                 return 0;
3032         }
3033
3034         /* OR_I. */
3035         src_val = strtoull(src, &src, 0);
3036         CHECK(!src[0], EINVAL);
3037
3038         if (dst[0] == 'h')
3039                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3040
3041         instr->type = INSTR_ALU_OR_I;
3042         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3043         instr->alu.dst.n_bits = fdst->n_bits;
3044         instr->alu.dst.offset = fdst->offset / 8;
3045         instr->alu.src_val = src_val;
3046         return 0;
3047 }
3048
3049 static int
3050 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3051                         struct action *action,
3052                         char **tokens,
3053                         int n_tokens,
3054                         struct instruction *instr,
3055                         struct instruction_data *data __rte_unused)
3056 {
3057         char *dst = tokens[1], *src = tokens[2];
3058         struct field *fdst, *fsrc;
3059         uint64_t src_val;
3060         uint32_t dst_struct_id = 0, src_struct_id = 0;
3061
3062         CHECK(n_tokens == 3, EINVAL);
3063
3064         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3065         CHECK(fdst, EINVAL);
3066         CHECK(!fdst->var_size, EINVAL);
3067
3068         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3069         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3070         if (fsrc) {
3071                 CHECK(!fsrc->var_size, EINVAL);
3072
3073                 instr->type = INSTR_ALU_XOR;
3074                 if (dst[0] != 'h' && src[0] == 'h')
3075                         instr->type = INSTR_ALU_XOR_MH;
3076                 if (dst[0] == 'h' && src[0] != 'h')
3077                         instr->type = INSTR_ALU_XOR_HM;
3078                 if (dst[0] == 'h' && src[0] == 'h')
3079                         instr->type = INSTR_ALU_XOR_HH;
3080
3081                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3082                 instr->alu.dst.n_bits = fdst->n_bits;
3083                 instr->alu.dst.offset = fdst->offset / 8;
3084                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3085                 instr->alu.src.n_bits = fsrc->n_bits;
3086                 instr->alu.src.offset = fsrc->offset / 8;
3087                 return 0;
3088         }
3089
3090         /* XOR_I. */
3091         src_val = strtoull(src, &src, 0);
3092         CHECK(!src[0], EINVAL);
3093
3094         if (dst[0] == 'h')
3095                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3096
3097         instr->type = INSTR_ALU_XOR_I;
3098         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3099         instr->alu.dst.n_bits = fdst->n_bits;
3100         instr->alu.dst.offset = fdst->offset / 8;
3101         instr->alu.src_val = src_val;
3102         return 0;
3103 }
3104
3105 static inline void
3106 instr_alu_add_exec(struct rte_swx_pipeline *p)
3107 {
3108         struct thread *t = &p->threads[p->thread_id];
3109         struct instruction *ip = t->ip;
3110
3111         TRACE("[Thread %2u] add\n", p->thread_id);
3112
3113         /* Structs. */
3114         ALU(t, ip, +);
3115
3116         /* Thread. */
3117         thread_ip_inc(p);
3118 }
3119
3120 static inline void
3121 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3122 {
3123         struct thread *t = &p->threads[p->thread_id];
3124         struct instruction *ip = t->ip;
3125
3126         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3127
3128         /* Structs. */
3129         ALU_MH(t, ip, +);
3130
3131         /* Thread. */
3132         thread_ip_inc(p);
3133 }
3134
3135 static inline void
3136 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3137 {
3138         struct thread *t = &p->threads[p->thread_id];
3139         struct instruction *ip = t->ip;
3140
3141         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3142
3143         /* Structs. */
3144         ALU_HM(t, ip, +);
3145
3146         /* Thread. */
3147         thread_ip_inc(p);
3148 }
3149
3150 static inline void
3151 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3152 {
3153         struct thread *t = &p->threads[p->thread_id];
3154         struct instruction *ip = t->ip;
3155
3156         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3157
3158         /* Structs. */
3159         ALU_HH(t, ip, +);
3160
3161         /* Thread. */
3162         thread_ip_inc(p);
3163 }
3164
3165 static inline void
3166 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3167 {
3168         struct thread *t = &p->threads[p->thread_id];
3169         struct instruction *ip = t->ip;
3170
3171         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3172
3173         /* Structs. */
3174         ALU_MI(t, ip, +);
3175
3176         /* Thread. */
3177         thread_ip_inc(p);
3178 }
3179
3180 static inline void
3181 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3182 {
3183         struct thread *t = &p->threads[p->thread_id];
3184         struct instruction *ip = t->ip;
3185
3186         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3187
3188         /* Structs. */
3189         ALU_HI(t, ip, +);
3190
3191         /* Thread. */
3192         thread_ip_inc(p);
3193 }
3194
3195 static inline void
3196 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3197 {
3198         struct thread *t = &p->threads[p->thread_id];
3199         struct instruction *ip = t->ip;
3200
3201         TRACE("[Thread %2u] sub\n", p->thread_id);
3202
3203         /* Structs. */
3204         ALU(t, ip, -);
3205
3206         /* Thread. */
3207         thread_ip_inc(p);
3208 }
3209
3210 static inline void
3211 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3212 {
3213         struct thread *t = &p->threads[p->thread_id];
3214         struct instruction *ip = t->ip;
3215
3216         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3217
3218         /* Structs. */
3219         ALU_MH(t, ip, -);
3220
3221         /* Thread. */
3222         thread_ip_inc(p);
3223 }
3224
3225 static inline void
3226 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3227 {
3228         struct thread *t = &p->threads[p->thread_id];
3229         struct instruction *ip = t->ip;
3230
3231         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3232
3233         /* Structs. */
3234         ALU_HM(t, ip, -);
3235
3236         /* Thread. */
3237         thread_ip_inc(p);
3238 }
3239
3240 static inline void
3241 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3242 {
3243         struct thread *t = &p->threads[p->thread_id];
3244         struct instruction *ip = t->ip;
3245
3246         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3247
3248         /* Structs. */
3249         ALU_HH(t, ip, -);
3250
3251         /* Thread. */
3252         thread_ip_inc(p);
3253 }
3254
3255 static inline void
3256 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3257 {
3258         struct thread *t = &p->threads[p->thread_id];
3259         struct instruction *ip = t->ip;
3260
3261         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3262
3263         /* Structs. */
3264         ALU_MI(t, ip, -);
3265
3266         /* Thread. */
3267         thread_ip_inc(p);
3268 }
3269
3270 static inline void
3271 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3272 {
3273         struct thread *t = &p->threads[p->thread_id];
3274         struct instruction *ip = t->ip;
3275
3276         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3277
3278         /* Structs. */
3279         ALU_HI(t, ip, -);
3280
3281         /* Thread. */
3282         thread_ip_inc(p);
3283 }
3284
3285 static inline void
3286 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3287 {
3288         struct thread *t = &p->threads[p->thread_id];
3289         struct instruction *ip = t->ip;
3290
3291         TRACE("[Thread %2u] shl\n", p->thread_id);
3292
3293         /* Structs. */
3294         ALU(t, ip, <<);
3295
3296         /* Thread. */
3297         thread_ip_inc(p);
3298 }
3299
3300 static inline void
3301 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3302 {
3303         struct thread *t = &p->threads[p->thread_id];
3304         struct instruction *ip = t->ip;
3305
3306         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3307
3308         /* Structs. */
3309         ALU_MH(t, ip, <<);
3310
3311         /* Thread. */
3312         thread_ip_inc(p);
3313 }
3314
3315 static inline void
3316 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3317 {
3318         struct thread *t = &p->threads[p->thread_id];
3319         struct instruction *ip = t->ip;
3320
3321         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3322
3323         /* Structs. */
3324         ALU_HM(t, ip, <<);
3325
3326         /* Thread. */
3327         thread_ip_inc(p);
3328 }
3329
3330 static inline void
3331 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3332 {
3333         struct thread *t = &p->threads[p->thread_id];
3334         struct instruction *ip = t->ip;
3335
3336         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3337
3338         /* Structs. */
3339         ALU_HH(t, ip, <<);
3340
3341         /* Thread. */
3342         thread_ip_inc(p);
3343 }
3344
3345 static inline void
3346 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3347 {
3348         struct thread *t = &p->threads[p->thread_id];
3349         struct instruction *ip = t->ip;
3350
3351         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3352
3353         /* Structs. */
3354         ALU_MI(t, ip, <<);
3355
3356         /* Thread. */
3357         thread_ip_inc(p);
3358 }
3359
3360 static inline void
3361 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3362 {
3363         struct thread *t = &p->threads[p->thread_id];
3364         struct instruction *ip = t->ip;
3365
3366         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3367
3368         /* Structs. */
3369         ALU_HI(t, ip, <<);
3370
3371         /* Thread. */
3372         thread_ip_inc(p);
3373 }
3374
3375 static inline void
3376 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3377 {
3378         struct thread *t = &p->threads[p->thread_id];
3379         struct instruction *ip = t->ip;
3380
3381         TRACE("[Thread %2u] shr\n", p->thread_id);
3382
3383         /* Structs. */
3384         ALU(t, ip, >>);
3385
3386         /* Thread. */
3387         thread_ip_inc(p);
3388 }
3389
3390 static inline void
3391 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3392 {
3393         struct thread *t = &p->threads[p->thread_id];
3394         struct instruction *ip = t->ip;
3395
3396         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3397
3398         /* Structs. */
3399         ALU_MH(t, ip, >>);
3400
3401         /* Thread. */
3402         thread_ip_inc(p);
3403 }
3404
3405 static inline void
3406 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3407 {
3408         struct thread *t = &p->threads[p->thread_id];
3409         struct instruction *ip = t->ip;
3410
3411         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3412
3413         /* Structs. */
3414         ALU_HM(t, ip, >>);
3415
3416         /* Thread. */
3417         thread_ip_inc(p);
3418 }
3419
3420 static inline void
3421 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3422 {
3423         struct thread *t = &p->threads[p->thread_id];
3424         struct instruction *ip = t->ip;
3425
3426         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3427
3428         /* Structs. */
3429         ALU_HH(t, ip, >>);
3430
3431         /* Thread. */
3432         thread_ip_inc(p);
3433 }
3434
3435 static inline void
3436 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3437 {
3438         struct thread *t = &p->threads[p->thread_id];
3439         struct instruction *ip = t->ip;
3440
3441         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3442
3443         /* Structs. */
3444         ALU_MI(t, ip, >>);
3445
3446         /* Thread. */
3447         thread_ip_inc(p);
3448 }
3449
3450 static inline void
3451 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455
3456         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3457
3458         /* Structs. */
3459         ALU_HI(t, ip, >>);
3460
3461         /* Thread. */
3462         thread_ip_inc(p);
3463 }
3464
3465 static inline void
3466 instr_alu_and_exec(struct rte_swx_pipeline *p)
3467 {
3468         struct thread *t = &p->threads[p->thread_id];
3469         struct instruction *ip = t->ip;
3470
3471         TRACE("[Thread %2u] and\n", p->thread_id);
3472
3473         /* Structs. */
3474         ALU(t, ip, &);
3475
3476         /* Thread. */
3477         thread_ip_inc(p);
3478 }
3479
3480 static inline void
3481 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3482 {
3483         struct thread *t = &p->threads[p->thread_id];
3484         struct instruction *ip = t->ip;
3485
3486         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3487
3488         /* Structs. */
3489         ALU_MH(t, ip, &);
3490
3491         /* Thread. */
3492         thread_ip_inc(p);
3493 }
3494
3495 static inline void
3496 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3497 {
3498         struct thread *t = &p->threads[p->thread_id];
3499         struct instruction *ip = t->ip;
3500
3501         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3502
3503         /* Structs. */
3504         ALU_HM_FAST(t, ip, &);
3505
3506         /* Thread. */
3507         thread_ip_inc(p);
3508 }
3509
3510 static inline void
3511 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3512 {
3513         struct thread *t = &p->threads[p->thread_id];
3514         struct instruction *ip = t->ip;
3515
3516         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3517
3518         /* Structs. */
3519         ALU_HH_FAST(t, ip, &);
3520
3521         /* Thread. */
3522         thread_ip_inc(p);
3523 }
3524
3525 static inline void
3526 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3527 {
3528         struct thread *t = &p->threads[p->thread_id];
3529         struct instruction *ip = t->ip;
3530
3531         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3532
3533         /* Structs. */
3534         ALU_I(t, ip, &);
3535
3536         /* Thread. */
3537         thread_ip_inc(p);
3538 }
3539
3540 static inline void
3541 instr_alu_or_exec(struct rte_swx_pipeline *p)
3542 {
3543         struct thread *t = &p->threads[p->thread_id];
3544         struct instruction *ip = t->ip;
3545
3546         TRACE("[Thread %2u] or\n", p->thread_id);
3547
3548         /* Structs. */
3549         ALU(t, ip, |);
3550
3551         /* Thread. */
3552         thread_ip_inc(p);
3553 }
3554
3555 static inline void
3556 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3557 {
3558         struct thread *t = &p->threads[p->thread_id];
3559         struct instruction *ip = t->ip;
3560
3561         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3562
3563         /* Structs. */
3564         ALU_MH(t, ip, |);
3565
3566         /* Thread. */
3567         thread_ip_inc(p);
3568 }
3569
3570 static inline void
3571 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3572 {
3573         struct thread *t = &p->threads[p->thread_id];
3574         struct instruction *ip = t->ip;
3575
3576         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3577
3578         /* Structs. */
3579         ALU_HM_FAST(t, ip, |);
3580
3581         /* Thread. */
3582         thread_ip_inc(p);
3583 }
3584
3585 static inline void
3586 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3587 {
3588         struct thread *t = &p->threads[p->thread_id];
3589         struct instruction *ip = t->ip;
3590
3591         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3592
3593         /* Structs. */
3594         ALU_HH_FAST(t, ip, |);
3595
3596         /* Thread. */
3597         thread_ip_inc(p);
3598 }
3599
3600 static inline void
3601 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3602 {
3603         struct thread *t = &p->threads[p->thread_id];
3604         struct instruction *ip = t->ip;
3605
3606         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3607
3608         /* Structs. */
3609         ALU_I(t, ip, |);
3610
3611         /* Thread. */
3612         thread_ip_inc(p);
3613 }
3614
3615 static inline void
3616 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3617 {
3618         struct thread *t = &p->threads[p->thread_id];
3619         struct instruction *ip = t->ip;
3620
3621         TRACE("[Thread %2u] xor\n", p->thread_id);
3622
3623         /* Structs. */
3624         ALU(t, ip, ^);
3625
3626         /* Thread. */
3627         thread_ip_inc(p);
3628 }
3629
3630 static inline void
3631 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3632 {
3633         struct thread *t = &p->threads[p->thread_id];
3634         struct instruction *ip = t->ip;
3635
3636         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3637
3638         /* Structs. */
3639         ALU_MH(t, ip, ^);
3640
3641         /* Thread. */
3642         thread_ip_inc(p);
3643 }
3644
3645 static inline void
3646 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3647 {
3648         struct thread *t = &p->threads[p->thread_id];
3649         struct instruction *ip = t->ip;
3650
3651         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3652
3653         /* Structs. */
3654         ALU_HM_FAST(t, ip, ^);
3655
3656         /* Thread. */
3657         thread_ip_inc(p);
3658 }
3659
3660 static inline void
3661 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3662 {
3663         struct thread *t = &p->threads[p->thread_id];
3664         struct instruction *ip = t->ip;
3665
3666         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3667
3668         /* Structs. */
3669         ALU_HH_FAST(t, ip, ^);
3670
3671         /* Thread. */
3672         thread_ip_inc(p);
3673 }
3674
3675 static inline void
3676 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3677 {
3678         struct thread *t = &p->threads[p->thread_id];
3679         struct instruction *ip = t->ip;
3680
3681         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3682
3683         /* Structs. */
3684         ALU_I(t, ip, ^);
3685
3686         /* Thread. */
3687         thread_ip_inc(p);
3688 }
3689
3690 static inline void
3691 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3692 {
3693         struct thread *t = &p->threads[p->thread_id];
3694         struct instruction *ip = t->ip;
3695         uint8_t *dst_struct, *src_struct;
3696         uint16_t *dst16_ptr, dst;
3697         uint64_t *src64_ptr, src64, src64_mask, src;
3698         uint64_t r;
3699
3700         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3701
3702         /* Structs. */
3703         dst_struct = t->structs[ip->alu.dst.struct_id];
3704         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3705         dst = *dst16_ptr;
3706
3707         src_struct = t->structs[ip->alu.src.struct_id];
3708         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3709         src64 = *src64_ptr;
3710         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3711         src = src64 & src64_mask;
3712
3713         r = dst;
3714         r = ~r & 0xFFFF;
3715
3716         /* The first input (r) is a 16-bit number. The second and the third
3717          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3718          * three numbers (output r) is a 34-bit number.
3719          */
3720         r += (src >> 32) + (src & 0xFFFFFFFF);
3721
3722         /* The first input is a 16-bit number. The second input is an 18-bit
3723          * number. In the worst case scenario, the sum of the two numbers is a
3724          * 19-bit number.
3725          */
3726         r = (r & 0xFFFF) + (r >> 16);
3727
3728         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3729          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3730          */
3731         r = (r & 0xFFFF) + (r >> 16);
3732
3733         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3734          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3735          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3736          * therefore the output r is always a 16-bit number.
3737          */
3738         r = (r & 0xFFFF) + (r >> 16);
3739
3740         r = ~r & 0xFFFF;
3741         r = r ? r : 0xFFFF;
3742
3743         *dst16_ptr = (uint16_t)r;
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 static inline void
3750 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3751 {
3752         struct thread *t = &p->threads[p->thread_id];
3753         struct instruction *ip = t->ip;
3754         uint8_t *dst_struct, *src_struct;
3755         uint16_t *dst16_ptr, dst;
3756         uint64_t *src64_ptr, src64, src64_mask, src;
3757         uint64_t r;
3758
3759         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3760
3761         /* Structs. */
3762         dst_struct = t->structs[ip->alu.dst.struct_id];
3763         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3764         dst = *dst16_ptr;
3765
3766         src_struct = t->structs[ip->alu.src.struct_id];
3767         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3768         src64 = *src64_ptr;
3769         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3770         src = src64 & src64_mask;
3771
3772         r = dst;
3773         r = ~r & 0xFFFF;
3774
3775         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3776          * the following sequence of operations in 2's complement arithmetic:
3777          *    a '- b = (a - b) % 0xFFFF.
3778          *
3779          * In order to prevent an underflow for the below subtraction, in which
3780          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3781          * minuend), we first add a multiple of the 0xFFFF modulus to the
3782          * minuend. The number we add to the minuend needs to be a 34-bit number
3783          * or higher, so for readability reasons we picked the 36-bit multiple.
3784          * We are effectively turning the 16-bit minuend into a 36-bit number:
3785          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3786          */
3787         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3788
3789         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3790          * result (the output r) is a 36-bit number.
3791          */
3792         r -= (src >> 32) + (src & 0xFFFFFFFF);
3793
3794         /* The first input is a 16-bit number. The second input is a 20-bit
3795          * number. Their sum is a 21-bit number.
3796          */
3797         r = (r & 0xFFFF) + (r >> 16);
3798
3799         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3800          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3801          */
3802         r = (r & 0xFFFF) + (r >> 16);
3803
3804         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3805          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3806          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3807          * generated, therefore the output r is always a 16-bit number.
3808          */
3809         r = (r & 0xFFFF) + (r >> 16);
3810
3811         r = ~r & 0xFFFF;
3812         r = r ? r : 0xFFFF;
3813
3814         *dst16_ptr = (uint16_t)r;
3815
3816         /* Thread. */
3817         thread_ip_inc(p);
3818 }
3819
3820 static inline void
3821 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3822 {
3823         struct thread *t = &p->threads[p->thread_id];
3824         struct instruction *ip = t->ip;
3825         uint8_t *dst_struct, *src_struct;
3826         uint16_t *dst16_ptr;
3827         uint32_t *src32_ptr;
3828         uint64_t r0, r1;
3829
3830         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3831
3832         /* Structs. */
3833         dst_struct = t->structs[ip->alu.dst.struct_id];
3834         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3835
3836         src_struct = t->structs[ip->alu.src.struct_id];
3837         src32_ptr = (uint32_t *)&src_struct[0];
3838
3839         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3840         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3841         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3842         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3843         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3844
3845         /* The first input is a 16-bit number. The second input is a 19-bit
3846          * number. Their sum is a 20-bit number.
3847          */
3848         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3849
3850         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3851          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3852          */
3853         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3854
3855         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3856          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3857          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3858          * generated, therefore the output r is always a 16-bit number.
3859          */
3860         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3861
3862         r0 = ~r0 & 0xFFFF;
3863         r0 = r0 ? r0 : 0xFFFF;
3864
3865         *dst16_ptr = (uint16_t)r0;
3866
3867         /* Thread. */
3868         thread_ip_inc(p);
3869 }
3870
3871 static inline void
3872 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3873 {
3874         struct thread *t = &p->threads[p->thread_id];
3875         struct instruction *ip = t->ip;
3876         uint8_t *dst_struct, *src_struct;
3877         uint16_t *dst16_ptr;
3878         uint32_t *src32_ptr;
3879         uint64_t r = 0;
3880         uint32_t i;
3881
3882         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3883
3884         /* Structs. */
3885         dst_struct = t->structs[ip->alu.dst.struct_id];
3886         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3887
3888         src_struct = t->structs[ip->alu.src.struct_id];
3889         src32_ptr = (uint32_t *)&src_struct[0];
3890
3891         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3892          * Therefore, in the worst case scenario, a 35-bit number is added to a
3893          * 16-bit number (the input r), so the output r is 36-bit number.
3894          */
3895         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3896                 r += *src32_ptr;
3897
3898         /* The first input is a 16-bit number. The second input is a 20-bit
3899          * number. Their sum is a 21-bit number.
3900          */
3901         r = (r & 0xFFFF) + (r >> 16);
3902
3903         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3904          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3905          */
3906         r = (r & 0xFFFF) + (r >> 16);
3907
3908         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3909          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3910          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3911          * generated, therefore the output r is always a 16-bit number.
3912          */
3913         r = (r & 0xFFFF) + (r >> 16);
3914
3915         r = ~r & 0xFFFF;
3916         r = r ? r : 0xFFFF;
3917
3918         *dst16_ptr = (uint16_t)r;
3919
3920         /* Thread. */
3921         thread_ip_inc(p);
3922 }
3923
3924 /*
3925  * Register array.
3926  */
3927 static struct regarray *
3928 regarray_find(struct rte_swx_pipeline *p, const char *name);
3929
3930 static int
3931 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3932                       struct action *action,
3933                       char **tokens,
3934                       int n_tokens,
3935                       struct instruction *instr,
3936                       struct instruction_data *data __rte_unused)
3937 {
3938         char *regarray = tokens[1], *idx = tokens[2];
3939         struct regarray *r;
3940         struct field *fidx;
3941         uint32_t idx_struct_id, idx_val;
3942
3943         CHECK(n_tokens == 3, EINVAL);
3944
3945         r = regarray_find(p, regarray);
3946         CHECK(r, EINVAL);
3947
3948         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3949         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3950         if (fidx) {
3951                 CHECK(!fidx->var_size, EINVAL);
3952
3953                 instr->type = INSTR_REGPREFETCH_RM;
3954                 if (idx[0] == 'h')
3955                         instr->type = INSTR_REGPREFETCH_RH;
3956
3957                 instr->regarray.regarray_id = r->id;
3958                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3959                 instr->regarray.idx.n_bits = fidx->n_bits;
3960                 instr->regarray.idx.offset = fidx->offset / 8;
3961                 instr->regarray.dstsrc_val = 0; /* Unused. */
3962                 return 0;
3963         }
3964
3965         /* REGPREFETCH_RI. */
3966         idx_val = strtoul(idx, &idx, 0);
3967         CHECK(!idx[0], EINVAL);
3968
3969         instr->type = INSTR_REGPREFETCH_RI;
3970         instr->regarray.regarray_id = r->id;
3971         instr->regarray.idx_val = idx_val;
3972         instr->regarray.dstsrc_val = 0; /* Unused. */
3973         return 0;
3974 }
3975
3976 static int
3977 instr_regrd_translate(struct rte_swx_pipeline *p,
3978                       struct action *action,
3979                       char **tokens,
3980                       int n_tokens,
3981                       struct instruction *instr,
3982                       struct instruction_data *data __rte_unused)
3983 {
3984         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3985         struct regarray *r;
3986         struct field *fdst, *fidx;
3987         uint32_t dst_struct_id, idx_struct_id, idx_val;
3988
3989         CHECK(n_tokens == 4, EINVAL);
3990
3991         r = regarray_find(p, regarray);
3992         CHECK(r, EINVAL);
3993
3994         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3995         CHECK(fdst, EINVAL);
3996         CHECK(!fdst->var_size, EINVAL);
3997
3998         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3999         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4000         if (fidx) {
4001                 CHECK(!fidx->var_size, EINVAL);
4002
4003                 instr->type = INSTR_REGRD_MRM;
4004                 if (dst[0] == 'h' && idx[0] != 'h')
4005                         instr->type = INSTR_REGRD_HRM;
4006                 if (dst[0] != 'h' && idx[0] == 'h')
4007                         instr->type = INSTR_REGRD_MRH;
4008                 if (dst[0] == 'h' && idx[0] == 'h')
4009                         instr->type = INSTR_REGRD_HRH;
4010
4011                 instr->regarray.regarray_id = r->id;
4012                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4013                 instr->regarray.idx.n_bits = fidx->n_bits;
4014                 instr->regarray.idx.offset = fidx->offset / 8;
4015                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4016                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4017                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4018                 return 0;
4019         }
4020
4021         /* REGRD_MRI, REGRD_HRI. */
4022         idx_val = strtoul(idx, &idx, 0);
4023         CHECK(!idx[0], EINVAL);
4024
4025         instr->type = INSTR_REGRD_MRI;
4026         if (dst[0] == 'h')
4027                 instr->type = INSTR_REGRD_HRI;
4028
4029         instr->regarray.regarray_id = r->id;
4030         instr->regarray.idx_val = idx_val;
4031         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4032         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4033         instr->regarray.dstsrc.offset = fdst->offset / 8;
4034         return 0;
4035 }
4036
4037 static int
4038 instr_regwr_translate(struct rte_swx_pipeline *p,
4039                       struct action *action,
4040                       char **tokens,
4041                       int n_tokens,
4042                       struct instruction *instr,
4043                       struct instruction_data *data __rte_unused)
4044 {
4045         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4046         struct regarray *r;
4047         struct field *fidx, *fsrc;
4048         uint64_t src_val;
4049         uint32_t idx_struct_id, idx_val, src_struct_id;
4050
4051         CHECK(n_tokens == 4, EINVAL);
4052
4053         r = regarray_find(p, regarray);
4054         CHECK(r, EINVAL);
4055
4056         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4057         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4058         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4059         if (fidx && fsrc) {
4060                 CHECK(!fidx->var_size, EINVAL);
4061                 CHECK(!fsrc->var_size, EINVAL);
4062
4063                 instr->type = INSTR_REGWR_RMM;
4064                 if (idx[0] == 'h' && src[0] != 'h')
4065                         instr->type = INSTR_REGWR_RHM;
4066                 if (idx[0] != 'h' && src[0] == 'h')
4067                         instr->type = INSTR_REGWR_RMH;
4068                 if (idx[0] == 'h' && src[0] == 'h')
4069                         instr->type = INSTR_REGWR_RHH;
4070
4071                 instr->regarray.regarray_id = r->id;
4072                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4073                 instr->regarray.idx.n_bits = fidx->n_bits;
4074                 instr->regarray.idx.offset = fidx->offset / 8;
4075                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4076                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4077                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4078                 return 0;
4079         }
4080
4081         /* REGWR_RHI, REGWR_RMI. */
4082         if (fidx && !fsrc) {
4083                 CHECK(!fidx->var_size, EINVAL);
4084
4085                 src_val = strtoull(src, &src, 0);
4086                 CHECK(!src[0], EINVAL);
4087
4088                 instr->type = INSTR_REGWR_RMI;
4089                 if (idx[0] == 'h')
4090                         instr->type = INSTR_REGWR_RHI;
4091
4092                 instr->regarray.regarray_id = r->id;
4093                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4094                 instr->regarray.idx.n_bits = fidx->n_bits;
4095                 instr->regarray.idx.offset = fidx->offset / 8;
4096                 instr->regarray.dstsrc_val = src_val;
4097                 return 0;
4098         }
4099
4100         /* REGWR_RIH, REGWR_RIM. */
4101         if (!fidx && fsrc) {
4102                 idx_val = strtoul(idx, &idx, 0);
4103                 CHECK(!idx[0], EINVAL);
4104
4105                 CHECK(!fsrc->var_size, EINVAL);
4106
4107                 instr->type = INSTR_REGWR_RIM;
4108                 if (src[0] == 'h')
4109                         instr->type = INSTR_REGWR_RIH;
4110
4111                 instr->regarray.regarray_id = r->id;
4112                 instr->regarray.idx_val = idx_val;
4113                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4114                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4115                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4116                 return 0;
4117         }
4118
4119         /* REGWR_RII. */
4120         src_val = strtoull(src, &src, 0);
4121         CHECK(!src[0], EINVAL);
4122
4123         idx_val = strtoul(idx, &idx, 0);
4124         CHECK(!idx[0], EINVAL);
4125
4126         instr->type = INSTR_REGWR_RII;
4127         instr->regarray.idx_val = idx_val;
4128         instr->regarray.dstsrc_val = src_val;
4129
4130         return 0;
4131 }
4132
4133 static int
4134 instr_regadd_translate(struct rte_swx_pipeline *p,
4135                        struct action *action,
4136                        char **tokens,
4137                        int n_tokens,
4138                        struct instruction *instr,
4139                        struct instruction_data *data __rte_unused)
4140 {
4141         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4142         struct regarray *r;
4143         struct field *fidx, *fsrc;
4144         uint64_t src_val;
4145         uint32_t idx_struct_id, idx_val, src_struct_id;
4146
4147         CHECK(n_tokens == 4, EINVAL);
4148
4149         r = regarray_find(p, regarray);
4150         CHECK(r, EINVAL);
4151
4152         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4153         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4154         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4155         if (fidx && fsrc) {
4156                 CHECK(!fidx->var_size, EINVAL);
4157                 CHECK(!fsrc->var_size, EINVAL);
4158
4159                 instr->type = INSTR_REGADD_RMM;
4160                 if (idx[0] == 'h' && src[0] != 'h')
4161                         instr->type = INSTR_REGADD_RHM;
4162                 if (idx[0] != 'h' && src[0] == 'h')
4163                         instr->type = INSTR_REGADD_RMH;
4164                 if (idx[0] == 'h' && src[0] == 'h')
4165                         instr->type = INSTR_REGADD_RHH;
4166
4167                 instr->regarray.regarray_id = r->id;
4168                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4169                 instr->regarray.idx.n_bits = fidx->n_bits;
4170                 instr->regarray.idx.offset = fidx->offset / 8;
4171                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4172                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4173                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4174                 return 0;
4175         }
4176
4177         /* REGADD_RHI, REGADD_RMI. */
4178         if (fidx && !fsrc) {
4179                 CHECK(!fidx->var_size, EINVAL);
4180
4181                 src_val = strtoull(src, &src, 0);
4182                 CHECK(!src[0], EINVAL);
4183
4184                 instr->type = INSTR_REGADD_RMI;
4185                 if (idx[0] == 'h')
4186                         instr->type = INSTR_REGADD_RHI;
4187
4188                 instr->regarray.regarray_id = r->id;
4189                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4190                 instr->regarray.idx.n_bits = fidx->n_bits;
4191                 instr->regarray.idx.offset = fidx->offset / 8;
4192                 instr->regarray.dstsrc_val = src_val;
4193                 return 0;
4194         }
4195
4196         /* REGADD_RIH, REGADD_RIM. */
4197         if (!fidx && fsrc) {
4198                 idx_val = strtoul(idx, &idx, 0);
4199                 CHECK(!idx[0], EINVAL);
4200
4201                 CHECK(!fsrc->var_size, EINVAL);
4202
4203                 instr->type = INSTR_REGADD_RIM;
4204                 if (src[0] == 'h')
4205                         instr->type = INSTR_REGADD_RIH;
4206
4207                 instr->regarray.regarray_id = r->id;
4208                 instr->regarray.idx_val = idx_val;
4209                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4210                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4211                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4212                 return 0;
4213         }
4214
4215         /* REGADD_RII. */
4216         src_val = strtoull(src, &src, 0);
4217         CHECK(!src[0], EINVAL);
4218
4219         idx_val = strtoul(idx, &idx, 0);
4220         CHECK(!idx[0], EINVAL);
4221
4222         instr->type = INSTR_REGADD_RII;
4223         instr->regarray.idx_val = idx_val;
4224         instr->regarray.dstsrc_val = src_val;
4225         return 0;
4226 }
4227
4228 static inline uint64_t *
4229 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4230 {
4231         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4232         return r->regarray;
4233 }
4234
4235 static inline uint64_t
4236 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4237 {
4238         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4239
4240         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4241         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4242         uint64_t idx64 = *idx64_ptr;
4243         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4244         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4245
4246         return idx;
4247 }
4248
4249 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4250
4251 static inline uint64_t
4252 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4253 {
4254         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4255
4256         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4257         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4258         uint64_t idx64 = *idx64_ptr;
4259         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4260
4261         return idx;
4262 }
4263
4264 #else
4265
4266 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4267
4268 #endif
4269
4270 static inline uint64_t
4271 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4272 {
4273         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4274
4275         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4276
4277         return idx;
4278 }
4279
4280 static inline uint64_t
4281 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4282 {
4283         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4284         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4285         uint64_t src64 = *src64_ptr;
4286         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4287         uint64_t src = src64 & src64_mask;
4288
4289         return src;
4290 }
4291
4292 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4293
4294 static inline uint64_t
4295 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4296 {
4297         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4298         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4299         uint64_t src64 = *src64_ptr;
4300         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4301
4302         return src;
4303 }
4304
4305 #else
4306
4307 #define instr_regarray_src_nbo instr_regarray_src_hbo
4308
4309 #endif
4310
4311 static inline void
4312 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4313 {
4314         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4315         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4316         uint64_t dst64 = *dst64_ptr;
4317         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4318
4319         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4320
4321 }
4322
4323 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4324
4325 static inline void
4326 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4327 {
4328         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4329         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4330         uint64_t dst64 = *dst64_ptr;
4331         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4332
4333         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4334         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4335 }
4336
4337 #else
4338
4339 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4340
4341 #endif
4342
4343 static inline void
4344 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4345 {
4346         struct thread *t = &p->threads[p->thread_id];
4347         struct instruction *ip = t->ip;
4348         uint64_t *regarray, idx;
4349
4350         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4351
4352         /* Structs. */
4353         regarray = instr_regarray_regarray(p, ip);
4354         idx = instr_regarray_idx_nbo(p, t, ip);
4355         rte_prefetch0(&regarray[idx]);
4356
4357         /* Thread. */
4358         thread_ip_inc(p);
4359 }
4360
4361 static inline void
4362 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4363 {
4364         struct thread *t = &p->threads[p->thread_id];
4365         struct instruction *ip = t->ip;
4366         uint64_t *regarray, idx;
4367
4368         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4369
4370         /* Structs. */
4371         regarray = instr_regarray_regarray(p, ip);
4372         idx = instr_regarray_idx_hbo(p, t, ip);
4373         rte_prefetch0(&regarray[idx]);
4374
4375         /* Thread. */
4376         thread_ip_inc(p);
4377 }
4378
4379 static inline void
4380 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4381 {
4382         struct thread *t = &p->threads[p->thread_id];
4383         struct instruction *ip = t->ip;
4384         uint64_t *regarray, idx;
4385
4386         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4387
4388         /* Structs. */
4389         regarray = instr_regarray_regarray(p, ip);
4390         idx = instr_regarray_idx_imm(p, ip);
4391         rte_prefetch0(&regarray[idx]);
4392
4393         /* Thread. */
4394         thread_ip_inc(p);
4395 }
4396
4397 static inline void
4398 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4399 {
4400         struct thread *t = &p->threads[p->thread_id];
4401         struct instruction *ip = t->ip;
4402         uint64_t *regarray, idx;
4403
4404         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4405
4406         /* Structs. */
4407         regarray = instr_regarray_regarray(p, ip);
4408         idx = instr_regarray_idx_nbo(p, t, ip);
4409         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4410
4411         /* Thread. */
4412         thread_ip_inc(p);
4413 }
4414
4415 static inline void
4416 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4417 {
4418         struct thread *t = &p->threads[p->thread_id];
4419         struct instruction *ip = t->ip;
4420         uint64_t *regarray, idx;
4421
4422         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4423
4424         /* Structs. */
4425         regarray = instr_regarray_regarray(p, ip);
4426         idx = instr_regarray_idx_hbo(p, t, ip);
4427         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4428
4429         /* Thread. */
4430         thread_ip_inc(p);
4431 }
4432
4433 static inline void
4434 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4435 {
4436         struct thread *t = &p->threads[p->thread_id];
4437         struct instruction *ip = t->ip;
4438         uint64_t *regarray, idx;
4439
4440         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4441
4442         /* Structs. */
4443         regarray = instr_regarray_regarray(p, ip);
4444         idx = instr_regarray_idx_nbo(p, t, ip);
4445         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4446
4447         /* Thread. */
4448         thread_ip_inc(p);
4449 }
4450
4451 static inline void
4452 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4453 {
4454         struct thread *t = &p->threads[p->thread_id];
4455         struct instruction *ip = t->ip;
4456         uint64_t *regarray, idx;
4457
4458         /* Structs. */
4459         regarray = instr_regarray_regarray(p, ip);
4460         idx = instr_regarray_idx_hbo(p, t, ip);
4461         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4462
4463         /* Thread. */
4464         thread_ip_inc(p);
4465 }
4466
4467 static inline void
4468 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4469 {
4470         struct thread *t = &p->threads[p->thread_id];
4471         struct instruction *ip = t->ip;
4472         uint64_t *regarray, idx;
4473
4474         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4475
4476         /* Structs. */
4477         regarray = instr_regarray_regarray(p, ip);
4478         idx = instr_regarray_idx_imm(p, ip);
4479         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4480
4481         /* Thread. */
4482         thread_ip_inc(p);
4483 }
4484
4485 static inline void
4486 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4487 {
4488         struct thread *t = &p->threads[p->thread_id];
4489         struct instruction *ip = t->ip;
4490         uint64_t *regarray, idx;
4491
4492         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4493
4494         /* Structs. */
4495         regarray = instr_regarray_regarray(p, ip);
4496         idx = instr_regarray_idx_imm(p, ip);
4497         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4498
4499         /* Thread. */
4500         thread_ip_inc(p);
4501 }
4502
4503 static inline void
4504 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4505 {
4506         struct thread *t = &p->threads[p->thread_id];
4507         struct instruction *ip = t->ip;
4508         uint64_t *regarray, idx, src;
4509
4510         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4511
4512         /* Structs. */
4513         regarray = instr_regarray_regarray(p, ip);
4514         idx = instr_regarray_idx_nbo(p, t, ip);
4515         src = instr_regarray_src_nbo(t, ip);
4516         regarray[idx] = src;
4517
4518         /* Thread. */
4519         thread_ip_inc(p);
4520 }
4521
4522 static inline void
4523 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4524 {
4525         struct thread *t = &p->threads[p->thread_id];
4526         struct instruction *ip = t->ip;
4527         uint64_t *regarray, idx, src;
4528
4529         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4530
4531         /* Structs. */
4532         regarray = instr_regarray_regarray(p, ip);
4533         idx = instr_regarray_idx_nbo(p, t, ip);
4534         src = instr_regarray_src_hbo(t, ip);
4535         regarray[idx] = src;
4536
4537         /* Thread. */
4538         thread_ip_inc(p);
4539 }
4540
4541 static inline void
4542 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4543 {
4544         struct thread *t = &p->threads[p->thread_id];
4545         struct instruction *ip = t->ip;
4546         uint64_t *regarray, idx, src;
4547
4548         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4549
4550         /* Structs. */
4551         regarray = instr_regarray_regarray(p, ip);
4552         idx = instr_regarray_idx_hbo(p, t, ip);
4553         src = instr_regarray_src_nbo(t, ip);
4554         regarray[idx] = src;
4555
4556         /* Thread. */
4557         thread_ip_inc(p);
4558 }
4559
4560 static inline void
4561 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4562 {
4563         struct thread *t = &p->threads[p->thread_id];
4564         struct instruction *ip = t->ip;
4565         uint64_t *regarray, idx, src;
4566
4567         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4568
4569         /* Structs. */
4570         regarray = instr_regarray_regarray(p, ip);
4571         idx = instr_regarray_idx_hbo(p, t, ip);
4572         src = instr_regarray_src_hbo(t, ip);
4573         regarray[idx] = src;
4574
4575         /* Thread. */
4576         thread_ip_inc(p);
4577 }
4578
4579 static inline void
4580 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4581 {
4582         struct thread *t = &p->threads[p->thread_id];
4583         struct instruction *ip = t->ip;
4584         uint64_t *regarray, idx, src;
4585
4586         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4587
4588         /* Structs. */
4589         regarray = instr_regarray_regarray(p, ip);
4590         idx = instr_regarray_idx_nbo(p, t, ip);
4591         src = ip->regarray.dstsrc_val;
4592         regarray[idx] = src;
4593
4594         /* Thread. */
4595         thread_ip_inc(p);
4596 }
4597
4598 static inline void
4599 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4600 {
4601         struct thread *t = &p->threads[p->thread_id];
4602         struct instruction *ip = t->ip;
4603         uint64_t *regarray, idx, src;
4604
4605         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4606
4607         /* Structs. */
4608         regarray = instr_regarray_regarray(p, ip);
4609         idx = instr_regarray_idx_hbo(p, t, ip);
4610         src = ip->regarray.dstsrc_val;
4611         regarray[idx] = src;
4612
4613         /* Thread. */
4614         thread_ip_inc(p);
4615 }
4616
4617 static inline void
4618 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4619 {
4620         struct thread *t = &p->threads[p->thread_id];
4621         struct instruction *ip = t->ip;
4622         uint64_t *regarray, idx, src;
4623
4624         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4625
4626         /* Structs. */
4627         regarray = instr_regarray_regarray(p, ip);
4628         idx = instr_regarray_idx_imm(p, ip);
4629         src = instr_regarray_src_nbo(t, ip);
4630         regarray[idx] = src;
4631
4632         /* Thread. */
4633         thread_ip_inc(p);
4634 }
4635
4636 static inline void
4637 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4638 {
4639         struct thread *t = &p->threads[p->thread_id];
4640         struct instruction *ip = t->ip;
4641         uint64_t *regarray, idx, src;
4642
4643         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4644
4645         /* Structs. */
4646         regarray = instr_regarray_regarray(p, ip);
4647         idx = instr_regarray_idx_imm(p, ip);
4648         src = instr_regarray_src_hbo(t, ip);
4649         regarray[idx] = src;
4650
4651         /* Thread. */
4652         thread_ip_inc(p);
4653 }
4654
4655 static inline void
4656 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4657 {
4658         struct thread *t = &p->threads[p->thread_id];
4659         struct instruction *ip = t->ip;
4660         uint64_t *regarray, idx, src;
4661
4662         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4663
4664         /* Structs. */
4665         regarray = instr_regarray_regarray(p, ip);
4666         idx = instr_regarray_idx_imm(p, ip);
4667         src = ip->regarray.dstsrc_val;
4668         regarray[idx] = src;
4669
4670         /* Thread. */
4671         thread_ip_inc(p);
4672 }
4673
4674 static inline void
4675 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4676 {
4677         struct thread *t = &p->threads[p->thread_id];
4678         struct instruction *ip = t->ip;
4679         uint64_t *regarray, idx, src;
4680
4681         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4682
4683         /* Structs. */
4684         regarray = instr_regarray_regarray(p, ip);
4685         idx = instr_regarray_idx_nbo(p, t, ip);
4686         src = instr_regarray_src_nbo(t, ip);
4687         regarray[idx] += src;
4688
4689         /* Thread. */
4690         thread_ip_inc(p);
4691 }
4692
4693 static inline void
4694 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4695 {
4696         struct thread *t = &p->threads[p->thread_id];
4697         struct instruction *ip = t->ip;
4698         uint64_t *regarray, idx, src;
4699
4700         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4701
4702         /* Structs. */
4703         regarray = instr_regarray_regarray(p, ip);
4704         idx = instr_regarray_idx_nbo(p, t, ip);
4705         src = instr_regarray_src_hbo(t, ip);
4706         regarray[idx] += src;
4707
4708         /* Thread. */
4709         thread_ip_inc(p);
4710 }
4711
4712 static inline void
4713 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4714 {
4715         struct thread *t = &p->threads[p->thread_id];
4716         struct instruction *ip = t->ip;
4717         uint64_t *regarray, idx, src;
4718
4719         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4720
4721         /* Structs. */
4722         regarray = instr_regarray_regarray(p, ip);
4723         idx = instr_regarray_idx_hbo(p, t, ip);
4724         src = instr_regarray_src_nbo(t, ip);
4725         regarray[idx] += src;
4726
4727         /* Thread. */
4728         thread_ip_inc(p);
4729 }
4730
4731 static inline void
4732 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4733 {
4734         struct thread *t = &p->threads[p->thread_id];
4735         struct instruction *ip = t->ip;
4736         uint64_t *regarray, idx, src;
4737
4738         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4739
4740         /* Structs. */
4741         regarray = instr_regarray_regarray(p, ip);
4742         idx = instr_regarray_idx_hbo(p, t, ip);
4743         src = instr_regarray_src_hbo(t, ip);
4744         regarray[idx] += src;
4745
4746         /* Thread. */
4747         thread_ip_inc(p);
4748 }
4749
4750 static inline void
4751 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4752 {
4753         struct thread *t = &p->threads[p->thread_id];
4754         struct instruction *ip = t->ip;
4755         uint64_t *regarray, idx, src;
4756
4757         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4758
4759         /* Structs. */
4760         regarray = instr_regarray_regarray(p, ip);
4761         idx = instr_regarray_idx_nbo(p, t, ip);
4762         src = ip->regarray.dstsrc_val;
4763         regarray[idx] += src;
4764
4765         /* Thread. */
4766         thread_ip_inc(p);
4767 }
4768
4769 static inline void
4770 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4771 {
4772         struct thread *t = &p->threads[p->thread_id];
4773         struct instruction *ip = t->ip;
4774         uint64_t *regarray, idx, src;
4775
4776         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4777
4778         /* Structs. */
4779         regarray = instr_regarray_regarray(p, ip);
4780         idx = instr_regarray_idx_hbo(p, t, ip);
4781         src = ip->regarray.dstsrc_val;
4782         regarray[idx] += src;
4783
4784         /* Thread. */
4785         thread_ip_inc(p);
4786 }
4787
4788 static inline void
4789 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4790 {
4791         struct thread *t = &p->threads[p->thread_id];
4792         struct instruction *ip = t->ip;
4793         uint64_t *regarray, idx, src;
4794
4795         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4796
4797         /* Structs. */
4798         regarray = instr_regarray_regarray(p, ip);
4799         idx = instr_regarray_idx_imm(p, ip);
4800         src = instr_regarray_src_nbo(t, ip);
4801         regarray[idx] += src;
4802
4803         /* Thread. */
4804         thread_ip_inc(p);
4805 }
4806
4807 static inline void
4808 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4809 {
4810         struct thread *t = &p->threads[p->thread_id];
4811         struct instruction *ip = t->ip;
4812         uint64_t *regarray, idx, src;
4813
4814         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4815
4816         /* Structs. */
4817         regarray = instr_regarray_regarray(p, ip);
4818         idx = instr_regarray_idx_imm(p, ip);
4819         src = instr_regarray_src_hbo(t, ip);
4820         regarray[idx] += src;
4821
4822         /* Thread. */
4823         thread_ip_inc(p);
4824 }
4825
4826 static inline void
4827 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4828 {
4829         struct thread *t = &p->threads[p->thread_id];
4830         struct instruction *ip = t->ip;
4831         uint64_t *regarray, idx, src;
4832
4833         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4834
4835         /* Structs. */
4836         regarray = instr_regarray_regarray(p, ip);
4837         idx = instr_regarray_idx_imm(p, ip);
4838         src = ip->regarray.dstsrc_val;
4839         regarray[idx] += src;
4840
4841         /* Thread. */
4842         thread_ip_inc(p);
4843 }
4844
4845 /*
4846  * metarray.
4847  */
4848 static struct metarray *
4849 metarray_find(struct rte_swx_pipeline *p, const char *name);
4850
4851 static int
4852 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4853                             struct action *action,
4854                             char **tokens,
4855                             int n_tokens,
4856                             struct instruction *instr,
4857                             struct instruction_data *data __rte_unused)
4858 {
4859         char *metarray = tokens[1], *idx = tokens[2];
4860         struct metarray *m;
4861         struct field *fidx;
4862         uint32_t idx_struct_id, idx_val;
4863
4864         CHECK(n_tokens == 3, EINVAL);
4865
4866         m = metarray_find(p, metarray);
4867         CHECK(m, EINVAL);
4868
4869         /* METPREFETCH_H, METPREFETCH_M. */
4870         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4871         if (fidx) {
4872                 CHECK(!fidx->var_size, EINVAL);
4873
4874                 instr->type = INSTR_METPREFETCH_M;
4875                 if (idx[0] == 'h')
4876                         instr->type = INSTR_METPREFETCH_H;
4877
4878                 instr->meter.metarray_id = m->id;
4879                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4880                 instr->meter.idx.n_bits = fidx->n_bits;
4881                 instr->meter.idx.offset = fidx->offset / 8;
4882                 return 0;
4883         }
4884
4885         /* METPREFETCH_I. */
4886         idx_val = strtoul(idx, &idx, 0);
4887         CHECK(!idx[0], EINVAL);
4888
4889         instr->type = INSTR_METPREFETCH_I;
4890         instr->meter.metarray_id = m->id;
4891         instr->meter.idx_val = idx_val;
4892         return 0;
4893 }
4894
4895 static int
4896 instr_meter_translate(struct rte_swx_pipeline *p,
4897                       struct action *action,
4898                       char **tokens,
4899                       int n_tokens,
4900                       struct instruction *instr,
4901                       struct instruction_data *data __rte_unused)
4902 {
4903         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4904         char *color_in = tokens[4], *color_out = tokens[5];
4905         struct metarray *m;
4906         struct field *fidx, *flength, *fcin, *fcout;
4907         uint32_t idx_struct_id, length_struct_id;
4908         uint32_t color_in_struct_id, color_out_struct_id;
4909
4910         CHECK(n_tokens == 6, EINVAL);
4911
4912         m = metarray_find(p, metarray);
4913         CHECK(m, EINVAL);
4914
4915         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4916
4917         flength = struct_field_parse(p, action, length, &length_struct_id);
4918         CHECK(flength, EINVAL);
4919         CHECK(!flength->var_size, EINVAL);
4920
4921         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4922
4923         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4924         CHECK(fcout, EINVAL);
4925         CHECK(!fcout->var_size, EINVAL);
4926
4927         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4928         if (fidx && fcin) {
4929                 CHECK(!fidx->var_size, EINVAL);
4930                 CHECK(!fcin->var_size, EINVAL);
4931
4932                 instr->type = INSTR_METER_MMM;
4933                 if (idx[0] == 'h' && length[0] == 'h')
4934                         instr->type = INSTR_METER_HHM;
4935                 if (idx[0] == 'h' && length[0] != 'h')
4936                         instr->type = INSTR_METER_HMM;
4937                 if (idx[0] != 'h' && length[0] == 'h')
4938                         instr->type = INSTR_METER_MHM;
4939
4940                 instr->meter.metarray_id = m->id;
4941
4942                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4943                 instr->meter.idx.n_bits = fidx->n_bits;
4944                 instr->meter.idx.offset = fidx->offset / 8;
4945
4946                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4947                 instr->meter.length.n_bits = flength->n_bits;
4948                 instr->meter.length.offset = flength->offset / 8;
4949
4950                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4951                 instr->meter.color_in.n_bits = fcin->n_bits;
4952                 instr->meter.color_in.offset = fcin->offset / 8;
4953
4954                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4955                 instr->meter.color_out.n_bits = fcout->n_bits;
4956                 instr->meter.color_out.offset = fcout->offset / 8;
4957
4958                 return 0;
4959         }
4960
4961         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4962         if (fidx && !fcin) {
4963                 uint32_t color_in_val;
4964
4965                 CHECK(!fidx->var_size, EINVAL);
4966
4967                 color_in_val = strtoul(color_in, &color_in, 0);
4968                 CHECK(!color_in[0], EINVAL);
4969
4970                 instr->type = INSTR_METER_MMI;
4971                 if (idx[0] == 'h' && length[0] == 'h')
4972                         instr->type = INSTR_METER_HHI;
4973                 if (idx[0] == 'h' && length[0] != 'h')
4974                         instr->type = INSTR_METER_HMI;
4975                 if (idx[0] != 'h' && length[0] == 'h')
4976                         instr->type = INSTR_METER_MHI;
4977
4978                 instr->meter.metarray_id = m->id;
4979
4980                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4981                 instr->meter.idx.n_bits = fidx->n_bits;
4982                 instr->meter.idx.offset = fidx->offset / 8;
4983
4984                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4985                 instr->meter.length.n_bits = flength->n_bits;
4986                 instr->meter.length.offset = flength->offset / 8;
4987
4988                 instr->meter.color_in_val = color_in_val;
4989
4990                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4991                 instr->meter.color_out.n_bits = fcout->n_bits;
4992                 instr->meter.color_out.offset = fcout->offset / 8;
4993
4994                 return 0;
4995         }
4996
4997         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4998         if (!fidx && fcin) {
4999                 uint32_t idx_val;
5000
5001                 idx_val = strtoul(idx, &idx, 0);
5002                 CHECK(!idx[0], EINVAL);
5003
5004                 CHECK(!fcin->var_size, EINVAL);
5005
5006                 instr->type = INSTR_METER_IMM;
5007                 if (length[0] == 'h')
5008                         instr->type = INSTR_METER_IHM;
5009
5010                 instr->meter.metarray_id = m->id;
5011
5012                 instr->meter.idx_val = idx_val;
5013
5014                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5015                 instr->meter.length.n_bits = flength->n_bits;
5016                 instr->meter.length.offset = flength->offset / 8;
5017
5018                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5019                 instr->meter.color_in.n_bits = fcin->n_bits;
5020                 instr->meter.color_in.offset = fcin->offset / 8;
5021
5022                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5023                 instr->meter.color_out.n_bits = fcout->n_bits;
5024                 instr->meter.color_out.offset = fcout->offset / 8;
5025
5026                 return 0;
5027         }
5028
5029         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5030         if (!fidx && !fcin) {
5031                 uint32_t idx_val, color_in_val;
5032
5033                 idx_val = strtoul(idx, &idx, 0);
5034                 CHECK(!idx[0], EINVAL);
5035
5036                 color_in_val = strtoul(color_in, &color_in, 0);
5037                 CHECK(!color_in[0], EINVAL);
5038
5039                 instr->type = INSTR_METER_IMI;
5040                 if (length[0] == 'h')
5041                         instr->type = INSTR_METER_IHI;
5042
5043                 instr->meter.metarray_id = m->id;
5044
5045                 instr->meter.idx_val = idx_val;
5046
5047                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5048                 instr->meter.length.n_bits = flength->n_bits;
5049                 instr->meter.length.offset = flength->offset / 8;
5050
5051                 instr->meter.color_in_val = color_in_val;
5052
5053                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5054                 instr->meter.color_out.n_bits = fcout->n_bits;
5055                 instr->meter.color_out.offset = fcout->offset / 8;
5056
5057                 return 0;
5058         }
5059
5060         CHECK(0, EINVAL);
5061 }
5062
5063 static inline struct meter *
5064 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5065 {
5066         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5067
5068         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5069         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5070         uint64_t idx64 = *idx64_ptr;
5071         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5072         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5073
5074         return &r->metarray[idx];
5075 }
5076
5077 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5078
5079 static inline struct meter *
5080 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5081 {
5082         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5083
5084         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5085         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5086         uint64_t idx64 = *idx64_ptr;
5087         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5088
5089         return &r->metarray[idx];
5090 }
5091
5092 #else
5093
5094 #define instr_meter_idx_nbo instr_meter_idx_hbo
5095
5096 #endif
5097
5098 static inline struct meter *
5099 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5100 {
5101         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5102
5103         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5104
5105         return &r->metarray[idx];
5106 }
5107
5108 static inline uint32_t
5109 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5110 {
5111         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5112         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5113         uint64_t src64 = *src64_ptr;
5114         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5115         uint64_t src = src64 & src64_mask;
5116
5117         return (uint32_t)src;
5118 }
5119
5120 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5121
5122 static inline uint32_t
5123 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5124 {
5125         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5126         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5127         uint64_t src64 = *src64_ptr;
5128         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5129
5130         return (uint32_t)src;
5131 }
5132
5133 #else
5134
5135 #define instr_meter_length_nbo instr_meter_length_hbo
5136
5137 #endif
5138
5139 static inline enum rte_color
5140 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5141 {
5142         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5143         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5144         uint64_t src64 = *src64_ptr;
5145         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5146         uint64_t src = src64 & src64_mask;
5147
5148         return (enum rte_color)src;
5149 }
5150
5151 static inline void
5152 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5153 {
5154         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5155         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5156         uint64_t dst64 = *dst64_ptr;
5157         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5158
5159         uint64_t src = (uint64_t)color_out;
5160
5161         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5162 }
5163
5164 static inline void
5165 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5166 {
5167         struct thread *t = &p->threads[p->thread_id];
5168         struct instruction *ip = t->ip;
5169         struct meter *m;
5170
5171         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5172
5173         /* Structs. */
5174         m = instr_meter_idx_nbo(p, t, ip);
5175         rte_prefetch0(m);
5176
5177         /* Thread. */
5178         thread_ip_inc(p);
5179 }
5180
5181 static inline void
5182 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5183 {
5184         struct thread *t = &p->threads[p->thread_id];
5185         struct instruction *ip = t->ip;
5186         struct meter *m;
5187
5188         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5189
5190         /* Structs. */
5191         m = instr_meter_idx_hbo(p, t, ip);
5192         rte_prefetch0(m);
5193
5194         /* Thread. */
5195         thread_ip_inc(p);
5196 }
5197
5198 static inline void
5199 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5200 {
5201         struct thread *t = &p->threads[p->thread_id];
5202         struct instruction *ip = t->ip;
5203         struct meter *m;
5204
5205         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5206
5207         /* Structs. */
5208         m = instr_meter_idx_imm(p, ip);
5209         rte_prefetch0(m);
5210
5211         /* Thread. */
5212         thread_ip_inc(p);
5213 }
5214
5215 static inline void
5216 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5217 {
5218         struct thread *t = &p->threads[p->thread_id];
5219         struct instruction *ip = t->ip;
5220         struct meter *m;
5221         uint64_t time, n_pkts, n_bytes;
5222         uint32_t length;
5223         enum rte_color color_in, color_out;
5224
5225         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5226
5227         /* Structs. */
5228         m = instr_meter_idx_nbo(p, t, ip);
5229         rte_prefetch0(m->n_pkts);
5230         time = rte_get_tsc_cycles();
5231         length = instr_meter_length_nbo(t, ip);
5232         color_in = instr_meter_color_in_hbo(t, ip);
5233
5234         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5235                 &m->profile->profile,
5236                 time,
5237                 length,
5238                 color_in);
5239
5240         color_out &= m->color_mask;
5241
5242         n_pkts = m->n_pkts[color_out];
5243         n_bytes = m->n_bytes[color_out];
5244
5245         instr_meter_color_out_hbo_set(t, ip, color_out);
5246
5247         m->n_pkts[color_out] = n_pkts + 1;
5248         m->n_bytes[color_out] = n_bytes + length;
5249
5250         /* Thread. */
5251         thread_ip_inc(p);
5252 }
5253
5254 static inline void
5255 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5256 {
5257         struct thread *t = &p->threads[p->thread_id];
5258         struct instruction *ip = t->ip;
5259         struct meter *m;
5260         uint64_t time, n_pkts, n_bytes;
5261         uint32_t length;
5262         enum rte_color color_in, color_out;
5263
5264         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5265
5266         /* Structs. */
5267         m = instr_meter_idx_nbo(p, t, ip);
5268         rte_prefetch0(m->n_pkts);
5269         time = rte_get_tsc_cycles();
5270         length = instr_meter_length_nbo(t, ip);
5271         color_in = (enum rte_color)ip->meter.color_in_val;
5272
5273         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5274                 &m->profile->profile,
5275                 time,
5276                 length,
5277                 color_in);
5278
5279         color_out &= m->color_mask;
5280
5281         n_pkts = m->n_pkts[color_out];
5282         n_bytes = m->n_bytes[color_out];
5283
5284         instr_meter_color_out_hbo_set(t, ip, color_out);
5285
5286         m->n_pkts[color_out] = n_pkts + 1;
5287         m->n_bytes[color_out] = n_bytes + length;
5288
5289         /* Thread. */
5290         thread_ip_inc(p);
5291 }
5292
5293 static inline void
5294 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5295 {
5296         struct thread *t = &p->threads[p->thread_id];
5297         struct instruction *ip = t->ip;
5298         struct meter *m;
5299         uint64_t time, n_pkts, n_bytes;
5300         uint32_t length;
5301         enum rte_color color_in, color_out;
5302
5303         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5304
5305         /* Structs. */
5306         m = instr_meter_idx_nbo(p, t, ip);
5307         rte_prefetch0(m->n_pkts);
5308         time = rte_get_tsc_cycles();
5309         length = instr_meter_length_hbo(t, ip);
5310         color_in = instr_meter_color_in_hbo(t, ip);
5311
5312         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5313                 &m->profile->profile,
5314                 time,
5315                 length,
5316                 color_in);
5317
5318         color_out &= m->color_mask;
5319
5320         n_pkts = m->n_pkts[color_out];
5321         n_bytes = m->n_bytes[color_out];
5322
5323         instr_meter_color_out_hbo_set(t, ip, color_out);
5324
5325         m->n_pkts[color_out] = n_pkts + 1;
5326         m->n_bytes[color_out] = n_bytes + length;
5327
5328         /* Thread. */
5329         thread_ip_inc(p);
5330 }
5331 static inline void
5332 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5333 {
5334         struct thread *t = &p->threads[p->thread_id];
5335         struct instruction *ip = t->ip;
5336         struct meter *m;
5337         uint64_t time, n_pkts, n_bytes;
5338         uint32_t length;
5339         enum rte_color color_in, color_out;
5340
5341         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5342
5343         /* Structs. */
5344         m = instr_meter_idx_nbo(p, t, ip);
5345         rte_prefetch0(m->n_pkts);
5346         time = rte_get_tsc_cycles();
5347         length = instr_meter_length_hbo(t, ip);
5348         color_in = (enum rte_color)ip->meter.color_in_val;
5349
5350         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5351                 &m->profile->profile,
5352                 time,
5353                 length,
5354                 color_in);
5355
5356         color_out &= m->color_mask;
5357
5358         n_pkts = m->n_pkts[color_out];
5359         n_bytes = m->n_bytes[color_out];
5360
5361         instr_meter_color_out_hbo_set(t, ip, color_out);
5362
5363         m->n_pkts[color_out] = n_pkts + 1;
5364         m->n_bytes[color_out] = n_bytes + length;
5365
5366         /* Thread. */
5367         thread_ip_inc(p);
5368 }
5369
5370 static inline void
5371 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5372 {
5373         struct thread *t = &p->threads[p->thread_id];
5374         struct instruction *ip = t->ip;
5375         struct meter *m;
5376         uint64_t time, n_pkts, n_bytes;
5377         uint32_t length;
5378         enum rte_color color_in, color_out;
5379
5380         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5381
5382         /* Structs. */
5383         m = instr_meter_idx_hbo(p, t, ip);
5384         rte_prefetch0(m->n_pkts);
5385         time = rte_get_tsc_cycles();
5386         length = instr_meter_length_nbo(t, ip);
5387         color_in = instr_meter_color_in_hbo(t, ip);
5388
5389         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5390                 &m->profile->profile,
5391                 time,
5392                 length,
5393                 color_in);
5394
5395         color_out &= m->color_mask;
5396
5397         n_pkts = m->n_pkts[color_out];
5398         n_bytes = m->n_bytes[color_out];
5399
5400         instr_meter_color_out_hbo_set(t, ip, color_out);
5401
5402         m->n_pkts[color_out] = n_pkts + 1;
5403         m->n_bytes[color_out] = n_bytes + length;
5404
5405         /* Thread. */
5406         thread_ip_inc(p);
5407 }
5408
5409 static inline void
5410 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5411 {
5412         struct thread *t = &p->threads[p->thread_id];
5413         struct instruction *ip = t->ip;
5414         struct meter *m;
5415         uint64_t time, n_pkts, n_bytes;
5416         uint32_t length;
5417         enum rte_color color_in, color_out;
5418
5419         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5420
5421         /* Structs. */
5422         m = instr_meter_idx_hbo(p, t, ip);
5423         rte_prefetch0(m->n_pkts);
5424         time = rte_get_tsc_cycles();
5425         length = instr_meter_length_nbo(t, ip);
5426         color_in = (enum rte_color)ip->meter.color_in_val;
5427
5428         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5429                 &m->profile->profile,
5430                 time,
5431                 length,
5432                 color_in);
5433
5434         color_out &= m->color_mask;
5435
5436         n_pkts = m->n_pkts[color_out];
5437         n_bytes = m->n_bytes[color_out];
5438
5439         instr_meter_color_out_hbo_set(t, ip, color_out);
5440
5441         m->n_pkts[color_out] = n_pkts + 1;
5442         m->n_bytes[color_out] = n_bytes + length;
5443
5444         /* Thread. */
5445         thread_ip_inc(p);
5446 }
5447
5448 static inline void
5449 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5450 {
5451         struct thread *t = &p->threads[p->thread_id];
5452         struct instruction *ip = t->ip;
5453         struct meter *m;
5454         uint64_t time, n_pkts, n_bytes;
5455         uint32_t length;
5456         enum rte_color color_in, color_out;
5457
5458         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5459
5460         /* Structs. */
5461         m = instr_meter_idx_hbo(p, t, ip);
5462         rte_prefetch0(m->n_pkts);
5463         time = rte_get_tsc_cycles();
5464         length = instr_meter_length_hbo(t, ip);
5465         color_in = instr_meter_color_in_hbo(t, ip);
5466
5467         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5468                 &m->profile->profile,
5469                 time,
5470                 length,
5471                 color_in);
5472
5473         color_out &= m->color_mask;
5474
5475         n_pkts = m->n_pkts[color_out];
5476         n_bytes = m->n_bytes[color_out];
5477
5478         instr_meter_color_out_hbo_set(t, ip, color_out);
5479
5480         m->n_pkts[color_out] = n_pkts + 1;
5481         m->n_bytes[color_out] = n_bytes + length;
5482
5483         /* Thread. */
5484         thread_ip_inc(p);
5485 }
5486
5487 static inline void
5488 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5489 {
5490         struct thread *t = &p->threads[p->thread_id];
5491         struct instruction *ip = t->ip;
5492         struct meter *m;
5493         uint64_t time, n_pkts, n_bytes;
5494         uint32_t length;
5495         enum rte_color color_in, color_out;
5496
5497         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5498
5499         /* Structs. */
5500         m = instr_meter_idx_hbo(p, t, ip);
5501         rte_prefetch0(m->n_pkts);
5502         time = rte_get_tsc_cycles();
5503         length = instr_meter_length_hbo(t, ip);
5504         color_in = (enum rte_color)ip->meter.color_in_val;
5505
5506         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5507                 &m->profile->profile,
5508                 time,
5509                 length,
5510                 color_in);
5511
5512         color_out &= m->color_mask;
5513
5514         n_pkts = m->n_pkts[color_out];
5515         n_bytes = m->n_bytes[color_out];
5516
5517         instr_meter_color_out_hbo_set(t, ip, color_out);
5518
5519         m->n_pkts[color_out] = n_pkts + 1;
5520         m->n_bytes[color_out] = n_bytes + length;
5521
5522         /* Thread. */
5523         thread_ip_inc(p);
5524 }
5525
5526 static inline void
5527 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5528 {
5529         struct thread *t = &p->threads[p->thread_id];
5530         struct instruction *ip = t->ip;
5531         struct meter *m;
5532         uint64_t time, n_pkts, n_bytes;
5533         uint32_t length;
5534         enum rte_color color_in, color_out;
5535
5536         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5537
5538         /* Structs. */
5539         m = instr_meter_idx_imm(p, ip);
5540         rte_prefetch0(m->n_pkts);
5541         time = rte_get_tsc_cycles();
5542         length = instr_meter_length_nbo(t, ip);
5543         color_in = instr_meter_color_in_hbo(t, ip);
5544
5545         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5546                 &m->profile->profile,
5547                 time,
5548                 length,
5549                 color_in);
5550
5551         color_out &= m->color_mask;
5552
5553         n_pkts = m->n_pkts[color_out];
5554         n_bytes = m->n_bytes[color_out];
5555
5556         instr_meter_color_out_hbo_set(t, ip, color_out);
5557
5558         m->n_pkts[color_out] = n_pkts + 1;
5559         m->n_bytes[color_out] = n_bytes + length;
5560
5561         /* Thread. */
5562         thread_ip_inc(p);
5563 }
5564
5565 static inline void
5566 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5567 {
5568         struct thread *t = &p->threads[p->thread_id];
5569         struct instruction *ip = t->ip;
5570         struct meter *m;
5571         uint64_t time, n_pkts, n_bytes;
5572         uint32_t length;
5573         enum rte_color color_in, color_out;
5574
5575         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5576
5577         /* Structs. */
5578         m = instr_meter_idx_imm(p, ip);
5579         rte_prefetch0(m->n_pkts);
5580         time = rte_get_tsc_cycles();
5581         length = instr_meter_length_nbo(t, ip);
5582         color_in = (enum rte_color)ip->meter.color_in_val;
5583
5584         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5585                 &m->profile->profile,
5586                 time,
5587                 length,
5588                 color_in);
5589
5590         color_out &= m->color_mask;
5591
5592         n_pkts = m->n_pkts[color_out];
5593         n_bytes = m->n_bytes[color_out];
5594
5595         instr_meter_color_out_hbo_set(t, ip, color_out);
5596
5597         m->n_pkts[color_out] = n_pkts + 1;
5598         m->n_bytes[color_out] = n_bytes + length;
5599
5600         /* Thread. */
5601         thread_ip_inc(p);
5602 }
5603
5604 static inline void
5605 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5606 {
5607         struct thread *t = &p->threads[p->thread_id];
5608         struct instruction *ip = t->ip;
5609         struct meter *m;
5610         uint64_t time, n_pkts, n_bytes;
5611         uint32_t length;
5612         enum rte_color color_in, color_out;
5613
5614         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5615
5616         /* Structs. */
5617         m = instr_meter_idx_imm(p, ip);
5618         rte_prefetch0(m->n_pkts);
5619         time = rte_get_tsc_cycles();
5620         length = instr_meter_length_hbo(t, ip);
5621         color_in = instr_meter_color_in_hbo(t, ip);
5622
5623         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5624                 &m->profile->profile,
5625                 time,
5626                 length,
5627                 color_in);
5628
5629         color_out &= m->color_mask;
5630
5631         n_pkts = m->n_pkts[color_out];
5632         n_bytes = m->n_bytes[color_out];
5633
5634         instr_meter_color_out_hbo_set(t, ip, color_out);
5635
5636         m->n_pkts[color_out] = n_pkts + 1;
5637         m->n_bytes[color_out] = n_bytes + length;
5638
5639         /* Thread. */
5640         thread_ip_inc(p);
5641 }
5642 static inline void
5643 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5644 {
5645         struct thread *t = &p->threads[p->thread_id];
5646         struct instruction *ip = t->ip;
5647         struct meter *m;
5648         uint64_t time, n_pkts, n_bytes;
5649         uint32_t length;
5650         enum rte_color color_in, color_out;
5651
5652         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5653
5654         /* Structs. */
5655         m = instr_meter_idx_imm(p, ip);
5656         rte_prefetch0(m->n_pkts);
5657         time = rte_get_tsc_cycles();
5658         length = instr_meter_length_hbo(t, ip);
5659         color_in = (enum rte_color)ip->meter.color_in_val;
5660
5661         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5662                 &m->profile->profile,
5663                 time,
5664                 length,
5665                 color_in);
5666
5667         color_out &= m->color_mask;
5668
5669         n_pkts = m->n_pkts[color_out];
5670         n_bytes = m->n_bytes[color_out];
5671
5672         instr_meter_color_out_hbo_set(t, ip, color_out);
5673
5674         m->n_pkts[color_out] = n_pkts + 1;
5675         m->n_bytes[color_out] = n_bytes + length;
5676
5677         /* Thread. */
5678         thread_ip_inc(p);
5679 }
5680
5681 /*
5682  * jmp.
5683  */
5684 static int
5685 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5686                     struct action *action __rte_unused,
5687                     char **tokens,
5688                     int n_tokens,
5689                     struct instruction *instr,
5690                     struct instruction_data *data)
5691 {
5692         CHECK(n_tokens == 2, EINVAL);
5693
5694         strcpy(data->jmp_label, tokens[1]);
5695
5696         instr->type = INSTR_JMP;
5697         instr->jmp.ip = NULL; /* Resolved later. */
5698         return 0;
5699 }
5700
5701 static int
5702 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5703                           struct action *action __rte_unused,
5704                           char **tokens,
5705                           int n_tokens,
5706                           struct instruction *instr,
5707                           struct instruction_data *data)
5708 {
5709         struct header *h;
5710
5711         CHECK(n_tokens == 3, EINVAL);
5712
5713         strcpy(data->jmp_label, tokens[1]);
5714
5715         h = header_parse(p, tokens[2]);
5716         CHECK(h, EINVAL);
5717
5718         instr->type = INSTR_JMP_VALID;
5719         instr->jmp.ip = NULL; /* Resolved later. */
5720         instr->jmp.header_id = h->id;
5721         return 0;
5722 }
5723
5724 static int
5725 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5726                             struct action *action __rte_unused,
5727                             char **tokens,
5728                             int n_tokens,
5729                             struct instruction *instr,
5730                             struct instruction_data *data)
5731 {
5732         struct header *h;
5733
5734         CHECK(n_tokens == 3, EINVAL);
5735
5736         strcpy(data->jmp_label, tokens[1]);
5737
5738         h = header_parse(p, tokens[2]);
5739         CHECK(h, EINVAL);
5740
5741         instr->type = INSTR_JMP_INVALID;
5742         instr->jmp.ip = NULL; /* Resolved later. */
5743         instr->jmp.header_id = h->id;
5744         return 0;
5745 }
5746
5747 static int
5748 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5749                         struct action *action,
5750                         char **tokens,
5751                         int n_tokens,
5752                         struct instruction *instr,
5753                         struct instruction_data *data)
5754 {
5755         CHECK(!action, EINVAL);
5756         CHECK(n_tokens == 2, EINVAL);
5757
5758         strcpy(data->jmp_label, tokens[1]);
5759
5760         instr->type = INSTR_JMP_HIT;
5761         instr->jmp.ip = NULL; /* Resolved later. */
5762         return 0;
5763 }
5764
5765 static int
5766 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5767                          struct action *action,
5768                          char **tokens,
5769                          int n_tokens,
5770                          struct instruction *instr,
5771                          struct instruction_data *data)
5772 {
5773         CHECK(!action, EINVAL);
5774         CHECK(n_tokens == 2, EINVAL);
5775
5776         strcpy(data->jmp_label, tokens[1]);
5777
5778         instr->type = INSTR_JMP_MISS;
5779         instr->jmp.ip = NULL; /* Resolved later. */
5780         return 0;
5781 }
5782
5783 static int
5784 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5785                                struct action *action,
5786                                char **tokens,
5787                                int n_tokens,
5788                                struct instruction *instr,
5789                                struct instruction_data *data)
5790 {
5791         struct action *a;
5792
5793         CHECK(!action, EINVAL);
5794         CHECK(n_tokens == 3, EINVAL);
5795
5796         strcpy(data->jmp_label, tokens[1]);
5797
5798         a = action_find(p, tokens[2]);
5799         CHECK(a, EINVAL);
5800
5801         instr->type = INSTR_JMP_ACTION_HIT;
5802         instr->jmp.ip = NULL; /* Resolved later. */
5803         instr->jmp.action_id = a->id;
5804         return 0;
5805 }
5806
5807 static int
5808 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5809                                 struct action *action,
5810                                 char **tokens,
5811                                 int n_tokens,
5812                                 struct instruction *instr,
5813                                 struct instruction_data *data)
5814 {
5815         struct action *a;
5816
5817         CHECK(!action, EINVAL);
5818         CHECK(n_tokens == 3, EINVAL);
5819
5820         strcpy(data->jmp_label, tokens[1]);
5821
5822         a = action_find(p, tokens[2]);
5823         CHECK(a, EINVAL);
5824
5825         instr->type = INSTR_JMP_ACTION_MISS;
5826         instr->jmp.ip = NULL; /* Resolved later. */
5827         instr->jmp.action_id = a->id;
5828         return 0;
5829 }
5830
5831 static int
5832 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5833                        struct action *action,
5834                        char **tokens,
5835                        int n_tokens,
5836                        struct instruction *instr,
5837                        struct instruction_data *data)
5838 {
5839         char *a = tokens[2], *b = tokens[3];
5840         struct field *fa, *fb;
5841         uint64_t b_val;
5842         uint32_t a_struct_id, b_struct_id;
5843
5844         CHECK(n_tokens == 4, EINVAL);
5845
5846         strcpy(data->jmp_label, tokens[1]);
5847
5848         fa = struct_field_parse(p, action, a, &a_struct_id);
5849         CHECK(fa, EINVAL);
5850         CHECK(!fa->var_size, EINVAL);
5851
5852         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5853         fb = struct_field_parse(p, action, b, &b_struct_id);
5854         if (fb) {
5855                 CHECK(!fb->var_size, EINVAL);
5856
5857                 instr->type = INSTR_JMP_EQ;
5858                 if (a[0] != 'h' && b[0] == 'h')
5859                         instr->type = INSTR_JMP_EQ_MH;
5860                 if (a[0] == 'h' && b[0] != 'h')
5861                         instr->type = INSTR_JMP_EQ_HM;
5862                 if (a[0] == 'h' && b[0] == 'h')
5863                         instr->type = INSTR_JMP_EQ_HH;
5864                 instr->jmp.ip = NULL; /* Resolved later. */
5865
5866                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5867                 instr->jmp.a.n_bits = fa->n_bits;
5868                 instr->jmp.a.offset = fa->offset / 8;
5869                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5870                 instr->jmp.b.n_bits = fb->n_bits;
5871                 instr->jmp.b.offset = fb->offset / 8;
5872                 return 0;
5873         }
5874
5875         /* JMP_EQ_I. */
5876         b_val = strtoull(b, &b, 0);
5877         CHECK(!b[0], EINVAL);
5878
5879         if (a[0] == 'h')
5880                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5881
5882         instr->type = INSTR_JMP_EQ_I;
5883         instr->jmp.ip = NULL; /* Resolved later. */
5884         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5885         instr->jmp.a.n_bits = fa->n_bits;
5886         instr->jmp.a.offset = fa->offset / 8;
5887         instr->jmp.b_val = b_val;
5888         return 0;
5889 }
5890
5891 static int
5892 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5893                         struct action *action,
5894                         char **tokens,
5895                         int n_tokens,
5896                         struct instruction *instr,
5897                         struct instruction_data *data)
5898 {
5899         char *a = tokens[2], *b = tokens[3];
5900         struct field *fa, *fb;
5901         uint64_t b_val;
5902         uint32_t a_struct_id, b_struct_id;
5903
5904         CHECK(n_tokens == 4, EINVAL);
5905
5906         strcpy(data->jmp_label, tokens[1]);
5907
5908         fa = struct_field_parse(p, action, a, &a_struct_id);
5909         CHECK(fa, EINVAL);
5910         CHECK(!fa->var_size, EINVAL);
5911
5912         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5913         fb = struct_field_parse(p, action, b, &b_struct_id);
5914         if (fb) {
5915                 CHECK(!fb->var_size, EINVAL);
5916
5917                 instr->type = INSTR_JMP_NEQ;
5918                 if (a[0] != 'h' && b[0] == 'h')
5919                         instr->type = INSTR_JMP_NEQ_MH;
5920                 if (a[0] == 'h' && b[0] != 'h')
5921                         instr->type = INSTR_JMP_NEQ_HM;
5922                 if (a[0] == 'h' && b[0] == 'h')
5923                         instr->type = INSTR_JMP_NEQ_HH;
5924                 instr->jmp.ip = NULL; /* Resolved later. */
5925
5926                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5927                 instr->jmp.a.n_bits = fa->n_bits;
5928                 instr->jmp.a.offset = fa->offset / 8;
5929                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5930                 instr->jmp.b.n_bits = fb->n_bits;
5931                 instr->jmp.b.offset = fb->offset / 8;
5932                 return 0;
5933         }
5934
5935         /* JMP_NEQ_I. */
5936         b_val = strtoull(b, &b, 0);
5937         CHECK(!b[0], EINVAL);
5938
5939         if (a[0] == 'h')
5940                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5941
5942         instr->type = INSTR_JMP_NEQ_I;
5943         instr->jmp.ip = NULL; /* Resolved later. */
5944         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5945         instr->jmp.a.n_bits = fa->n_bits;
5946         instr->jmp.a.offset = fa->offset / 8;
5947         instr->jmp.b_val = b_val;
5948         return 0;
5949 }
5950
5951 static int
5952 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5953                        struct action *action,
5954                        char **tokens,
5955                        int n_tokens,
5956                        struct instruction *instr,
5957                        struct instruction_data *data)
5958 {
5959         char *a = tokens[2], *b = tokens[3];
5960         struct field *fa, *fb;
5961         uint64_t b_val;
5962         uint32_t a_struct_id, b_struct_id;
5963
5964         CHECK(n_tokens == 4, EINVAL);
5965
5966         strcpy(data->jmp_label, tokens[1]);
5967
5968         fa = struct_field_parse(p, action, a, &a_struct_id);
5969         CHECK(fa, EINVAL);
5970         CHECK(!fa->var_size, EINVAL);
5971
5972         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5973         fb = struct_field_parse(p, action, b, &b_struct_id);
5974         if (fb) {
5975                 CHECK(!fb->var_size, EINVAL);
5976
5977                 instr->type = INSTR_JMP_LT;
5978                 if (a[0] == 'h' && b[0] != 'h')
5979                         instr->type = INSTR_JMP_LT_HM;
5980                 if (a[0] != 'h' && b[0] == 'h')
5981                         instr->type = INSTR_JMP_LT_MH;
5982                 if (a[0] == 'h' && b[0] == 'h')
5983                         instr->type = INSTR_JMP_LT_HH;
5984                 instr->jmp.ip = NULL; /* Resolved later. */
5985
5986                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5987                 instr->jmp.a.n_bits = fa->n_bits;
5988                 instr->jmp.a.offset = fa->offset / 8;
5989                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5990                 instr->jmp.b.n_bits = fb->n_bits;
5991                 instr->jmp.b.offset = fb->offset / 8;
5992                 return 0;
5993         }
5994
5995         /* JMP_LT_MI, JMP_LT_HI. */
5996         b_val = strtoull(b, &b, 0);
5997         CHECK(!b[0], EINVAL);
5998
5999         instr->type = INSTR_JMP_LT_MI;
6000         if (a[0] == 'h')
6001                 instr->type = INSTR_JMP_LT_HI;
6002         instr->jmp.ip = NULL; /* Resolved later. */
6003
6004         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6005         instr->jmp.a.n_bits = fa->n_bits;
6006         instr->jmp.a.offset = fa->offset / 8;
6007         instr->jmp.b_val = b_val;
6008         return 0;
6009 }
6010
6011 static int
6012 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6013                        struct action *action,
6014                        char **tokens,
6015                        int n_tokens,
6016                        struct instruction *instr,
6017                        struct instruction_data *data)
6018 {
6019         char *a = tokens[2], *b = tokens[3];
6020         struct field *fa, *fb;
6021         uint64_t b_val;
6022         uint32_t a_struct_id, b_struct_id;
6023
6024         CHECK(n_tokens == 4, EINVAL);
6025
6026         strcpy(data->jmp_label, tokens[1]);
6027
6028         fa = struct_field_parse(p, action, a, &a_struct_id);
6029         CHECK(fa, EINVAL);
6030         CHECK(!fa->var_size, EINVAL);
6031
6032         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6033         fb = struct_field_parse(p, action, b, &b_struct_id);
6034         if (fb) {
6035                 CHECK(!fb->var_size, EINVAL);
6036
6037                 instr->type = INSTR_JMP_GT;
6038                 if (a[0] == 'h' && b[0] != 'h')
6039                         instr->type = INSTR_JMP_GT_HM;
6040                 if (a[0] != 'h' && b[0] == 'h')
6041                         instr->type = INSTR_JMP_GT_MH;
6042                 if (a[0] == 'h' && b[0] == 'h')
6043                         instr->type = INSTR_JMP_GT_HH;
6044                 instr->jmp.ip = NULL; /* Resolved later. */
6045
6046                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6047                 instr->jmp.a.n_bits = fa->n_bits;
6048                 instr->jmp.a.offset = fa->offset / 8;
6049                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6050                 instr->jmp.b.n_bits = fb->n_bits;
6051                 instr->jmp.b.offset = fb->offset / 8;
6052                 return 0;
6053         }
6054
6055         /* JMP_GT_MI, JMP_GT_HI. */
6056         b_val = strtoull(b, &b, 0);
6057         CHECK(!b[0], EINVAL);
6058
6059         instr->type = INSTR_JMP_GT_MI;
6060         if (a[0] == 'h')
6061                 instr->type = INSTR_JMP_GT_HI;
6062         instr->jmp.ip = NULL; /* Resolved later. */
6063
6064         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6065         instr->jmp.a.n_bits = fa->n_bits;
6066         instr->jmp.a.offset = fa->offset / 8;
6067         instr->jmp.b_val = b_val;
6068         return 0;
6069 }
6070
6071 static inline void
6072 instr_jmp_exec(struct rte_swx_pipeline *p)
6073 {
6074         struct thread *t = &p->threads[p->thread_id];
6075         struct instruction *ip = t->ip;
6076
6077         TRACE("[Thread %2u] jmp\n", p->thread_id);
6078
6079         thread_ip_set(t, ip->jmp.ip);
6080 }
6081
6082 static inline void
6083 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6084 {
6085         struct thread *t = &p->threads[p->thread_id];
6086         struct instruction *ip = t->ip;
6087         uint32_t header_id = ip->jmp.header_id;
6088
6089         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6090
6091         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6092 }
6093
6094 static inline void
6095 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6096 {
6097         struct thread *t = &p->threads[p->thread_id];
6098         struct instruction *ip = t->ip;
6099         uint32_t header_id = ip->jmp.header_id;
6100
6101         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6102
6103         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6104 }
6105
6106 static inline void
6107 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6108 {
6109         struct thread *t = &p->threads[p->thread_id];
6110         struct instruction *ip = t->ip;
6111         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6112
6113         TRACE("[Thread %2u] jmph\n", p->thread_id);
6114
6115         t->ip = ip_next[t->hit];
6116 }
6117
6118 static inline void
6119 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6120 {
6121         struct thread *t = &p->threads[p->thread_id];
6122         struct instruction *ip = t->ip;
6123         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6124
6125         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6126
6127         t->ip = ip_next[t->hit];
6128 }
6129
6130 static inline void
6131 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6132 {
6133         struct thread *t = &p->threads[p->thread_id];
6134         struct instruction *ip = t->ip;
6135
6136         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6137
6138         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6139 }
6140
6141 static inline void
6142 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6143 {
6144         struct thread *t = &p->threads[p->thread_id];
6145         struct instruction *ip = t->ip;
6146
6147         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6148
6149         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6150 }
6151
6152 static inline void
6153 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6154 {
6155         struct thread *t = &p->threads[p->thread_id];
6156         struct instruction *ip = t->ip;
6157
6158         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6159
6160         JMP_CMP(t, ip, ==);
6161 }
6162
6163 static inline void
6164 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6165 {
6166         struct thread *t = &p->threads[p->thread_id];
6167         struct instruction *ip = t->ip;
6168
6169         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6170
6171         JMP_CMP_MH(t, ip, ==);
6172 }
6173
6174 static inline void
6175 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6176 {
6177         struct thread *t = &p->threads[p->thread_id];
6178         struct instruction *ip = t->ip;
6179
6180         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6181
6182         JMP_CMP_HM(t, ip, ==);
6183 }
6184
6185 static inline void
6186 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6187 {
6188         struct thread *t = &p->threads[p->thread_id];
6189         struct instruction *ip = t->ip;
6190
6191         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6192
6193         JMP_CMP_HH_FAST(t, ip, ==);
6194 }
6195
6196 static inline void
6197 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6198 {
6199         struct thread *t = &p->threads[p->thread_id];
6200         struct instruction *ip = t->ip;
6201
6202         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6203
6204         JMP_CMP_I(t, ip, ==);
6205 }
6206
6207 static inline void
6208 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6209 {
6210         struct thread *t = &p->threads[p->thread_id];
6211         struct instruction *ip = t->ip;
6212
6213         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6214
6215         JMP_CMP(t, ip, !=);
6216 }
6217
6218 static inline void
6219 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6220 {
6221         struct thread *t = &p->threads[p->thread_id];
6222         struct instruction *ip = t->ip;
6223
6224         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6225
6226         JMP_CMP_MH(t, ip, !=);
6227 }
6228
6229 static inline void
6230 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6231 {
6232         struct thread *t = &p->threads[p->thread_id];
6233         struct instruction *ip = t->ip;
6234
6235         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6236
6237         JMP_CMP_HM(t, ip, !=);
6238 }
6239
6240 static inline void
6241 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6242 {
6243         struct thread *t = &p->threads[p->thread_id];
6244         struct instruction *ip = t->ip;
6245
6246         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6247
6248         JMP_CMP_HH_FAST(t, ip, !=);
6249 }
6250
6251 static inline void
6252 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6253 {
6254         struct thread *t = &p->threads[p->thread_id];
6255         struct instruction *ip = t->ip;
6256
6257         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6258
6259         JMP_CMP_I(t, ip, !=);
6260 }
6261
6262 static inline void
6263 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6264 {
6265         struct thread *t = &p->threads[p->thread_id];
6266         struct instruction *ip = t->ip;
6267
6268         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6269
6270         JMP_CMP(t, ip, <);
6271 }
6272
6273 static inline void
6274 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6275 {
6276         struct thread *t = &p->threads[p->thread_id];
6277         struct instruction *ip = t->ip;
6278
6279         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6280
6281         JMP_CMP_MH(t, ip, <);
6282 }
6283
6284 static inline void
6285 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6286 {
6287         struct thread *t = &p->threads[p->thread_id];
6288         struct instruction *ip = t->ip;
6289
6290         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6291
6292         JMP_CMP_HM(t, ip, <);
6293 }
6294
6295 static inline void
6296 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6297 {
6298         struct thread *t = &p->threads[p->thread_id];
6299         struct instruction *ip = t->ip;
6300
6301         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6302
6303         JMP_CMP_HH(t, ip, <);
6304 }
6305
6306 static inline void
6307 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6308 {
6309         struct thread *t = &p->threads[p->thread_id];
6310         struct instruction *ip = t->ip;
6311
6312         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6313
6314         JMP_CMP_MI(t, ip, <);
6315 }
6316
6317 static inline void
6318 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6319 {
6320         struct thread *t = &p->threads[p->thread_id];
6321         struct instruction *ip = t->ip;
6322
6323         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6324
6325         JMP_CMP_HI(t, ip, <);
6326 }
6327
6328 static inline void
6329 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6330 {
6331         struct thread *t = &p->threads[p->thread_id];
6332         struct instruction *ip = t->ip;
6333
6334         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6335
6336         JMP_CMP(t, ip, >);
6337 }
6338
6339 static inline void
6340 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6341 {
6342         struct thread *t = &p->threads[p->thread_id];
6343         struct instruction *ip = t->ip;
6344
6345         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6346
6347         JMP_CMP_MH(t, ip, >);
6348 }
6349
6350 static inline void
6351 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6352 {
6353         struct thread *t = &p->threads[p->thread_id];
6354         struct instruction *ip = t->ip;
6355
6356         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6357
6358         JMP_CMP_HM(t, ip, >);
6359 }
6360
6361 static inline void
6362 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6363 {
6364         struct thread *t = &p->threads[p->thread_id];
6365         struct instruction *ip = t->ip;
6366
6367         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6368
6369         JMP_CMP_HH(t, ip, >);
6370 }
6371
6372 static inline void
6373 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6374 {
6375         struct thread *t = &p->threads[p->thread_id];
6376         struct instruction *ip = t->ip;
6377
6378         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6379
6380         JMP_CMP_MI(t, ip, >);
6381 }
6382
6383 static inline void
6384 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6385 {
6386         struct thread *t = &p->threads[p->thread_id];
6387         struct instruction *ip = t->ip;
6388
6389         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6390
6391         JMP_CMP_HI(t, ip, >);
6392 }
6393
6394 /*
6395  * return.
6396  */
6397 static int
6398 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6399                        struct action *action,
6400                        char **tokens __rte_unused,
6401                        int n_tokens,
6402                        struct instruction *instr,
6403                        struct instruction_data *data __rte_unused)
6404 {
6405         CHECK(action, EINVAL);
6406         CHECK(n_tokens == 1, EINVAL);
6407
6408         instr->type = INSTR_RETURN;
6409         return 0;
6410 }
6411
6412 static inline void
6413 instr_return_exec(struct rte_swx_pipeline *p)
6414 {
6415         struct thread *t = &p->threads[p->thread_id];
6416
6417         TRACE("[Thread %2u] return\n", p->thread_id);
6418
6419         t->ip = t->ret;
6420 }
6421
6422 static int
6423 instr_translate(struct rte_swx_pipeline *p,
6424                 struct action *action,
6425                 char *string,
6426                 struct instruction *instr,
6427                 struct instruction_data *data)
6428 {
6429         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6430         int n_tokens = 0, tpos = 0;
6431
6432         /* Parse the instruction string into tokens. */
6433         for ( ; ; ) {
6434                 char *token;
6435
6436                 token = strtok_r(string, " \t\v", &string);
6437                 if (!token)
6438                         break;
6439
6440                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6441                 CHECK_NAME(token, EINVAL);
6442
6443                 tokens[n_tokens] = token;
6444                 n_tokens++;
6445         }
6446
6447         CHECK(n_tokens, EINVAL);
6448
6449         /* Handle the optional instruction label. */
6450         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6451                 strcpy(data->label, tokens[0]);
6452
6453                 tpos += 2;
6454                 CHECK(n_tokens - tpos, EINVAL);
6455         }
6456
6457         /* Identify the instruction type. */
6458         if (!strcmp(tokens[tpos], "rx"))
6459                 return instr_rx_translate(p,
6460                                           action,
6461                                           &tokens[tpos],
6462                                           n_tokens - tpos,
6463                                           instr,
6464                                           data);
6465
6466         if (!strcmp(tokens[tpos], "tx"))
6467                 return instr_tx_translate(p,
6468                                           action,
6469                                           &tokens[tpos],
6470                                           n_tokens - tpos,
6471                                           instr,
6472                                           data);
6473
6474         if (!strcmp(tokens[tpos], "drop"))
6475                 return instr_drop_translate(p,
6476                                             action,
6477                                             &tokens[tpos],
6478                                             n_tokens - tpos,
6479                                             instr,
6480                                             data);
6481
6482         if (!strcmp(tokens[tpos], "extract"))
6483                 return instr_hdr_extract_translate(p,
6484                                                    action,
6485                                                    &tokens[tpos],
6486                                                    n_tokens - tpos,
6487                                                    instr,
6488                                                    data);
6489
6490         if (!strcmp(tokens[tpos], "lookahead"))
6491                 return instr_hdr_lookahead_translate(p,
6492                                                      action,
6493                                                      &tokens[tpos],
6494                                                      n_tokens - tpos,
6495                                                      instr,
6496                                                      data);
6497
6498         if (!strcmp(tokens[tpos], "emit"))
6499                 return instr_hdr_emit_translate(p,
6500                                                 action,
6501                                                 &tokens[tpos],
6502                                                 n_tokens - tpos,
6503                                                 instr,
6504                                                 data);
6505
6506         if (!strcmp(tokens[tpos], "validate"))
6507                 return instr_hdr_validate_translate(p,
6508                                                     action,
6509                                                     &tokens[tpos],
6510                                                     n_tokens - tpos,
6511                                                     instr,
6512                                                     data);
6513
6514         if (!strcmp(tokens[tpos], "invalidate"))
6515                 return instr_hdr_invalidate_translate(p,
6516                                                       action,
6517                                                       &tokens[tpos],
6518                                                       n_tokens - tpos,
6519                                                       instr,
6520                                                       data);
6521
6522         if (!strcmp(tokens[tpos], "mov"))
6523                 return instr_mov_translate(p,
6524                                            action,
6525                                            &tokens[tpos],
6526                                            n_tokens - tpos,
6527                                            instr,
6528                                            data);
6529
6530         if (!strcmp(tokens[tpos], "add"))
6531                 return instr_alu_add_translate(p,
6532                                                action,
6533                                                &tokens[tpos],
6534                                                n_tokens - tpos,
6535                                                instr,
6536                                                data);
6537
6538         if (!strcmp(tokens[tpos], "sub"))
6539                 return instr_alu_sub_translate(p,
6540                                                action,
6541                                                &tokens[tpos],
6542                                                n_tokens - tpos,
6543                                                instr,
6544                                                data);
6545
6546         if (!strcmp(tokens[tpos], "ckadd"))
6547                 return instr_alu_ckadd_translate(p,
6548                                                  action,
6549                                                  &tokens[tpos],
6550                                                  n_tokens - tpos,
6551                                                  instr,
6552                                                  data);
6553
6554         if (!strcmp(tokens[tpos], "cksub"))
6555                 return instr_alu_cksub_translate(p,
6556                                                  action,
6557                                                  &tokens[tpos],
6558                                                  n_tokens - tpos,
6559                                                  instr,
6560                                                  data);
6561
6562         if (!strcmp(tokens[tpos], "and"))
6563                 return instr_alu_and_translate(p,
6564                                                action,
6565                                                &tokens[tpos],
6566                                                n_tokens - tpos,
6567                                                instr,
6568                                                data);
6569
6570         if (!strcmp(tokens[tpos], "or"))
6571                 return instr_alu_or_translate(p,
6572                                               action,
6573                                               &tokens[tpos],
6574                                               n_tokens - tpos,
6575                                               instr,
6576                                               data);
6577
6578         if (!strcmp(tokens[tpos], "xor"))
6579                 return instr_alu_xor_translate(p,
6580                                                action,
6581                                                &tokens[tpos],
6582                                                n_tokens - tpos,
6583                                                instr,
6584                                                data);
6585
6586         if (!strcmp(tokens[tpos], "shl"))
6587                 return instr_alu_shl_translate(p,
6588                                                action,
6589                                                &tokens[tpos],
6590                                                n_tokens - tpos,
6591                                                instr,
6592                                                data);
6593
6594         if (!strcmp(tokens[tpos], "shr"))
6595                 return instr_alu_shr_translate(p,
6596                                                action,
6597                                                &tokens[tpos],
6598                                                n_tokens - tpos,
6599                                                instr,
6600                                                data);
6601
6602         if (!strcmp(tokens[tpos], "regprefetch"))
6603                 return instr_regprefetch_translate(p,
6604                                                    action,
6605                                                    &tokens[tpos],
6606                                                    n_tokens - tpos,
6607                                                    instr,
6608                                                    data);
6609
6610         if (!strcmp(tokens[tpos], "regrd"))
6611                 return instr_regrd_translate(p,
6612                                              action,
6613                                              &tokens[tpos],
6614                                              n_tokens - tpos,
6615                                              instr,
6616                                              data);
6617
6618         if (!strcmp(tokens[tpos], "regwr"))
6619                 return instr_regwr_translate(p,
6620                                              action,
6621                                              &tokens[tpos],
6622                                              n_tokens - tpos,
6623                                              instr,
6624                                              data);
6625
6626         if (!strcmp(tokens[tpos], "regadd"))
6627                 return instr_regadd_translate(p,
6628                                               action,
6629                                               &tokens[tpos],
6630                                               n_tokens - tpos,
6631                                               instr,
6632                                               data);
6633
6634         if (!strcmp(tokens[tpos], "metprefetch"))
6635                 return instr_metprefetch_translate(p,
6636                                                    action,
6637                                                    &tokens[tpos],
6638                                                    n_tokens - tpos,
6639                                                    instr,
6640                                                    data);
6641
6642         if (!strcmp(tokens[tpos], "meter"))
6643                 return instr_meter_translate(p,
6644                                              action,
6645                                              &tokens[tpos],
6646                                              n_tokens - tpos,
6647                                              instr,
6648                                              data);
6649
6650         if (!strcmp(tokens[tpos], "table"))
6651                 return instr_table_translate(p,
6652                                              action,
6653                                              &tokens[tpos],
6654                                              n_tokens - tpos,
6655                                              instr,
6656                                              data);
6657
6658         if (!strcmp(tokens[tpos], "learn"))
6659                 return instr_learn_translate(p,
6660                                              action,
6661                                              &tokens[tpos],
6662                                              n_tokens - tpos,
6663                                              instr,
6664                                              data);
6665
6666         if (!strcmp(tokens[tpos], "forget"))
6667                 return instr_forget_translate(p,
6668                                               action,
6669                                               &tokens[tpos],
6670                                               n_tokens - tpos,
6671                                               instr,
6672                                               data);
6673
6674         if (!strcmp(tokens[tpos], "extern"))
6675                 return instr_extern_translate(p,
6676                                               action,
6677                                               &tokens[tpos],
6678                                               n_tokens - tpos,
6679                                               instr,
6680                                               data);
6681
6682         if (!strcmp(tokens[tpos], "jmp"))
6683                 return instr_jmp_translate(p,
6684                                            action,
6685                                            &tokens[tpos],
6686                                            n_tokens - tpos,
6687                                            instr,
6688                                            data);
6689
6690         if (!strcmp(tokens[tpos], "jmpv"))
6691                 return instr_jmp_valid_translate(p,
6692                                                  action,
6693                                                  &tokens[tpos],
6694                                                  n_tokens - tpos,
6695                                                  instr,
6696                                                  data);
6697
6698         if (!strcmp(tokens[tpos], "jmpnv"))
6699                 return instr_jmp_invalid_translate(p,
6700                                                    action,
6701                                                    &tokens[tpos],
6702                                                    n_tokens - tpos,
6703                                                    instr,
6704                                                    data);
6705
6706         if (!strcmp(tokens[tpos], "jmph"))
6707                 return instr_jmp_hit_translate(p,
6708                                                action,
6709                                                &tokens[tpos],
6710                                                n_tokens - tpos,
6711                                                instr,
6712                                                data);
6713
6714         if (!strcmp(tokens[tpos], "jmpnh"))
6715                 return instr_jmp_miss_translate(p,
6716                                                 action,
6717                                                 &tokens[tpos],
6718                                                 n_tokens - tpos,
6719                                                 instr,
6720                                                 data);
6721
6722         if (!strcmp(tokens[tpos], "jmpa"))
6723                 return instr_jmp_action_hit_translate(p,
6724                                                       action,
6725                                                       &tokens[tpos],
6726                                                       n_tokens - tpos,
6727                                                       instr,
6728                                                       data);
6729
6730         if (!strcmp(tokens[tpos], "jmpna"))
6731                 return instr_jmp_action_miss_translate(p,
6732                                                        action,
6733                                                        &tokens[tpos],
6734                                                        n_tokens - tpos,
6735                                                        instr,
6736                                                        data);
6737
6738         if (!strcmp(tokens[tpos], "jmpeq"))
6739                 return instr_jmp_eq_translate(p,
6740                                               action,
6741                                               &tokens[tpos],
6742                                               n_tokens - tpos,
6743                                               instr,
6744                                               data);
6745
6746         if (!strcmp(tokens[tpos], "jmpneq"))
6747                 return instr_jmp_neq_translate(p,
6748                                                action,
6749                                                &tokens[tpos],
6750                                                n_tokens - tpos,
6751                                                instr,
6752                                                data);
6753
6754         if (!strcmp(tokens[tpos], "jmplt"))
6755                 return instr_jmp_lt_translate(p,
6756                                               action,
6757                                               &tokens[tpos],
6758                                               n_tokens - tpos,
6759                                               instr,
6760                                               data);
6761
6762         if (!strcmp(tokens[tpos], "jmpgt"))
6763                 return instr_jmp_gt_translate(p,
6764                                               action,
6765                                               &tokens[tpos],
6766                                               n_tokens - tpos,
6767                                               instr,
6768                                               data);
6769
6770         if (!strcmp(tokens[tpos], "return"))
6771                 return instr_return_translate(p,
6772                                               action,
6773                                               &tokens[tpos],
6774                                               n_tokens - tpos,
6775                                               instr,
6776                                               data);
6777
6778         CHECK(0, EINVAL);
6779 }
6780
6781 static struct instruction_data *
6782 label_find(struct instruction_data *data, uint32_t n, const char *label)
6783 {
6784         uint32_t i;
6785
6786         for (i = 0; i < n; i++)
6787                 if (!strcmp(label, data[i].label))
6788                         return &data[i];
6789
6790         return NULL;
6791 }
6792
6793 static uint32_t
6794 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6795 {
6796         uint32_t count = 0, i;
6797
6798         if (!label[0])
6799                 return 0;
6800
6801         for (i = 0; i < n; i++)
6802                 if (!strcmp(label, data[i].jmp_label))
6803                         count++;
6804
6805         return count;
6806 }
6807
6808 static int
6809 instr_label_check(struct instruction_data *instruction_data,
6810                   uint32_t n_instructions)
6811 {
6812         uint32_t i;
6813
6814         /* Check that all instruction labels are unique. */
6815         for (i = 0; i < n_instructions; i++) {
6816                 struct instruction_data *data = &instruction_data[i];
6817                 char *label = data->label;
6818                 uint32_t j;
6819
6820                 if (!label[0])
6821                         continue;
6822
6823                 for (j = i + 1; j < n_instructions; j++)
6824                         CHECK(strcmp(label, data[j].label), EINVAL);
6825         }
6826
6827         /* Get users for each instruction label. */
6828         for (i = 0; i < n_instructions; i++) {
6829                 struct instruction_data *data = &instruction_data[i];
6830                 char *label = data->label;
6831
6832                 data->n_users = label_is_used(instruction_data,
6833                                               n_instructions,
6834                                               label);
6835         }
6836
6837         return 0;
6838 }
6839
6840 static int
6841 instr_jmp_resolve(struct instruction *instructions,
6842                   struct instruction_data *instruction_data,
6843                   uint32_t n_instructions)
6844 {
6845         uint32_t i;
6846
6847         for (i = 0; i < n_instructions; i++) {
6848                 struct instruction *instr = &instructions[i];
6849                 struct instruction_data *data = &instruction_data[i];
6850                 struct instruction_data *found;
6851
6852                 if (!instruction_is_jmp(instr))
6853                         continue;
6854
6855                 found = label_find(instruction_data,
6856                                    n_instructions,
6857                                    data->jmp_label);
6858                 CHECK(found, EINVAL);
6859
6860                 instr->jmp.ip = &instructions[found - instruction_data];
6861         }
6862
6863         return 0;
6864 }
6865
6866 static int
6867 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6868              struct action *a,
6869              struct instruction *instr,
6870              struct instruction_data *data __rte_unused,
6871              uint32_t n_instructions)
6872 {
6873         if (!a) {
6874                 enum instruction_type type;
6875                 uint32_t i;
6876
6877                 /* Check that the first instruction is rx. */
6878                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6879
6880                 /* Check that there is at least one tx instruction. */
6881                 for (i = 0; i < n_instructions; i++) {
6882                         type = instr[i].type;
6883
6884                         if (instruction_is_tx(type))
6885                                 break;
6886                 }
6887                 CHECK(i < n_instructions, EINVAL);
6888
6889                 /* Check that the last instruction is either tx or unconditional
6890                  * jump.
6891                  */
6892                 type = instr[n_instructions - 1].type;
6893                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6894         }
6895
6896         if (a) {
6897                 enum instruction_type type;
6898                 uint32_t i;
6899
6900                 /* Check that there is at least one return or tx instruction. */
6901                 for (i = 0; i < n_instructions; i++) {
6902                         type = instr[i].type;
6903
6904                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6905                                 break;
6906                 }
6907                 CHECK(i < n_instructions, EINVAL);
6908         }
6909
6910         return 0;
6911 }
6912
6913 static uint32_t
6914 instr_compact(struct instruction *instructions,
6915               struct instruction_data *instruction_data,
6916               uint32_t n_instructions)
6917 {
6918         uint32_t i, pos = 0;
6919
6920         /* Eliminate the invalid instructions that have been optimized out. */
6921         for (i = 0; i < n_instructions; i++) {
6922                 struct instruction *instr = &instructions[i];
6923                 struct instruction_data *data = &instruction_data[i];
6924
6925                 if (data->invalid)
6926                         continue;
6927
6928                 if (i != pos) {
6929                         memcpy(&instructions[pos], instr, sizeof(*instr));
6930                         memcpy(&instruction_data[pos], data, sizeof(*data));
6931                 }
6932
6933                 pos++;
6934         }
6935
6936         return pos;
6937 }
6938
6939 static int
6940 instr_pattern_extract_many_search(struct instruction *instr,
6941                                   struct instruction_data *data,
6942                                   uint32_t n_instr,
6943                                   uint32_t *n_pattern_instr)
6944 {
6945         uint32_t i;
6946
6947         for (i = 0; i < n_instr; i++) {
6948                 if (data[i].invalid)
6949                         break;
6950
6951                 if (instr[i].type != INSTR_HDR_EXTRACT)
6952                         break;
6953
6954                 if (i == RTE_DIM(instr->io.hdr.header_id))
6955                         break;
6956
6957                 if (i && data[i].n_users)
6958                         break;
6959         }
6960
6961         if (i < 2)
6962                 return 0;
6963
6964         *n_pattern_instr = i;
6965         return 1;
6966 }
6967
6968 static void
6969 instr_pattern_extract_many_replace(struct instruction *instr,
6970                                    struct instruction_data *data,
6971                                    uint32_t n_instr)
6972 {
6973         uint32_t i;
6974
6975         for (i = 1; i < n_instr; i++) {
6976                 instr[0].type++;
6977                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6978                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6979                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6980
6981                 data[i].invalid = 1;
6982         }
6983 }
6984
6985 static uint32_t
6986 instr_pattern_extract_many_optimize(struct instruction *instructions,
6987                                     struct instruction_data *instruction_data,
6988                                     uint32_t n_instructions)
6989 {
6990         uint32_t i;
6991
6992         for (i = 0; i < n_instructions; ) {
6993                 struct instruction *instr = &instructions[i];
6994                 struct instruction_data *data = &instruction_data[i];
6995                 uint32_t n_instr = 0;
6996                 int detected;
6997
6998                 /* Extract many. */
6999                 detected = instr_pattern_extract_many_search(instr,
7000                                                              data,
7001                                                              n_instructions - i,
7002                                                              &n_instr);
7003                 if (detected) {
7004                         instr_pattern_extract_many_replace(instr,
7005                                                            data,
7006                                                            n_instr);
7007                         i += n_instr;
7008                         continue;
7009                 }
7010
7011                 /* No pattern starting at the current instruction. */
7012                 i++;
7013         }
7014
7015         /* Eliminate the invalid instructions that have been optimized out. */
7016         n_instructions = instr_compact(instructions,
7017                                        instruction_data,
7018                                        n_instructions);
7019
7020         return n_instructions;
7021 }
7022
7023 static int
7024 instr_pattern_emit_many_tx_search(struct instruction *instr,
7025                                   struct instruction_data *data,
7026                                   uint32_t n_instr,
7027                                   uint32_t *n_pattern_instr)
7028 {
7029         uint32_t i;
7030
7031         for (i = 0; i < n_instr; i++) {
7032                 if (data[i].invalid)
7033                         break;
7034
7035                 if (instr[i].type != INSTR_HDR_EMIT)
7036                         break;
7037
7038                 if (i == RTE_DIM(instr->io.hdr.header_id))
7039                         break;
7040
7041                 if (i && data[i].n_users)
7042                         break;
7043         }
7044
7045         if (!i)
7046                 return 0;
7047
7048         if (!instruction_is_tx(instr[i].type))
7049                 return 0;
7050
7051         if (data[i].n_users)
7052                 return 0;
7053
7054         i++;
7055
7056         *n_pattern_instr = i;
7057         return 1;
7058 }
7059
7060 static void
7061 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7062                                    struct instruction_data *data,
7063                                    uint32_t n_instr)
7064 {
7065         uint32_t i;
7066
7067         /* Any emit instruction in addition to the first one. */
7068         for (i = 1; i < n_instr - 1; i++) {
7069                 instr[0].type++;
7070                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7071                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7072                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7073
7074                 data[i].invalid = 1;
7075         }
7076
7077         /* The TX instruction is the last one in the pattern. */
7078         instr[0].type++;
7079         instr[0].io.io.offset = instr[i].io.io.offset;
7080         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7081         data[i].invalid = 1;
7082 }
7083
7084 static uint32_t
7085 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7086                                     struct instruction_data *instruction_data,
7087                                     uint32_t n_instructions)
7088 {
7089         uint32_t i;
7090
7091         for (i = 0; i < n_instructions; ) {
7092                 struct instruction *instr = &instructions[i];
7093                 struct instruction_data *data = &instruction_data[i];
7094                 uint32_t n_instr = 0;
7095                 int detected;
7096
7097                 /* Emit many + TX. */
7098                 detected = instr_pattern_emit_many_tx_search(instr,
7099                                                              data,
7100                                                              n_instructions - i,
7101                                                              &n_instr);
7102                 if (detected) {
7103                         instr_pattern_emit_many_tx_replace(instr,
7104                                                            data,
7105                                                            n_instr);
7106                         i += n_instr;
7107                         continue;
7108                 }
7109
7110                 /* No pattern starting at the current instruction. */
7111                 i++;
7112         }
7113
7114         /* Eliminate the invalid instructions that have been optimized out. */
7115         n_instructions = instr_compact(instructions,
7116                                        instruction_data,
7117                                        n_instructions);
7118
7119         return n_instructions;
7120 }
7121
7122 static uint32_t
7123 action_arg_src_mov_count(struct action *a,
7124                          uint32_t arg_id,
7125                          struct instruction *instructions,
7126                          struct instruction_data *instruction_data,
7127                          uint32_t n_instructions);
7128
7129 static int
7130 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7131                                       struct action *a,
7132                                       struct instruction *instr,
7133                                       struct instruction_data *data,
7134                                       uint32_t n_instr,
7135                                       struct instruction *instructions,
7136                                       struct instruction_data *instruction_data,
7137                                       uint32_t n_instructions,
7138                                       uint32_t *n_pattern_instr)
7139 {
7140         struct header *h;
7141         uint32_t src_field_id, i, j;
7142
7143         /* Prerequisites. */
7144         if (!a || !a->st)
7145                 return 0;
7146
7147         /* First instruction: MOV_HM. */
7148         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7149                 return 0;
7150
7151         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7152         if (!h || h->st->var_size)
7153                 return 0;
7154
7155         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7156                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7157                         break;
7158
7159         if (src_field_id == a->st->n_fields)
7160                 return 0;
7161
7162         if (instr[0].mov.dst.offset ||
7163             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7164             instr[0].mov.src.struct_id ||
7165             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7166             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7167                 return 0;
7168
7169         if ((n_instr < h->st->n_fields + 1) ||
7170              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7171                 return 0;
7172
7173         /* Subsequent instructions: MOV_HM. */
7174         for (i = 1; i < h->st->n_fields; i++)
7175                 if (data[i].invalid ||
7176                     data[i].n_users ||
7177                     (instr[i].type != INSTR_MOV_HM) ||
7178                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7179                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7180                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7181                     instr[i].mov.src.struct_id ||
7182                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7183                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7184                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7185                         return 0;
7186
7187         /* Last instruction: HDR_VALIDATE. */
7188         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7189             (instr[i].valid.header_id != h->id))
7190                 return 0;
7191
7192         /* Check that none of the action args that are used as source for this
7193          * DMA transfer are not used as source in any other mov instruction.
7194          */
7195         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7196                 uint32_t n_users;
7197
7198                 n_users = action_arg_src_mov_count(a,
7199                                                    j,
7200                                                    instructions,
7201                                                    instruction_data,
7202                                                    n_instructions);
7203                 if (n_users > 1)
7204                         return 0;
7205         }
7206
7207         *n_pattern_instr = 1 + i;
7208         return 1;
7209 }
7210
7211 static void
7212 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7213                                        struct action *a,
7214                                        struct instruction *instr,
7215                                        struct instruction_data *data,
7216                                        uint32_t n_instr)
7217 {
7218         struct header *h;
7219         uint32_t src_field_id, src_offset, i;
7220
7221         /* Read from the instructions before they are modified. */
7222         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7223         if (!h)
7224                 return;
7225
7226         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7227                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7228                         break;
7229
7230         if (src_field_id == a->st->n_fields)
7231                 return;
7232
7233         src_offset = instr[0].mov.src.offset;
7234
7235         /* Modify the instructions. */
7236         instr[0].type = INSTR_DMA_HT;
7237         instr[0].dma.dst.header_id[0] = h->id;
7238         instr[0].dma.dst.struct_id[0] = h->struct_id;
7239         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7240         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7241
7242         for (i = 1; i < n_instr; i++)
7243                 data[i].invalid = 1;
7244
7245         /* Update the endianness of the action arguments to header endianness. */
7246         for (i = 0; i < h->st->n_fields; i++)
7247                 a->args_endianness[src_field_id + i] = 1;
7248 }
7249
7250 static uint32_t
7251 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7252                                         struct action *a,
7253                                         struct instruction *instructions,
7254                                         struct instruction_data *instruction_data,
7255                                         uint32_t n_instructions)
7256 {
7257         uint32_t i;
7258
7259         if (!a || !a->st)
7260                 return n_instructions;
7261
7262         for (i = 0; i < n_instructions; ) {
7263                 struct instruction *instr = &instructions[i];
7264                 struct instruction_data *data = &instruction_data[i];
7265                 uint32_t n_instr = 0;
7266                 int detected;
7267
7268                 /* Mov all + validate. */
7269                 detected = instr_pattern_mov_all_validate_search(p,
7270                                                                  a,
7271                                                                  instr,
7272                                                                  data,
7273                                                                  n_instructions - i,
7274                                                                  instructions,
7275                                                                  instruction_data,
7276                                                                  n_instructions,
7277                                                                  &n_instr);
7278                 if (detected) {
7279                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7280                         i += n_instr;
7281                         continue;
7282                 }
7283
7284                 /* No pattern starting at the current instruction. */
7285                 i++;
7286         }
7287
7288         /* Eliminate the invalid instructions that have been optimized out. */
7289         n_instructions = instr_compact(instructions,
7290                                        instruction_data,
7291                                        n_instructions);
7292
7293         return n_instructions;
7294 }
7295
7296 static int
7297 instr_pattern_dma_many_search(struct instruction *instr,
7298                               struct instruction_data *data,
7299                               uint32_t n_instr,
7300                               uint32_t *n_pattern_instr)
7301 {
7302         uint32_t i;
7303
7304         for (i = 0; i < n_instr; i++) {
7305                 if (data[i].invalid)
7306                         break;
7307
7308                 if (instr[i].type != INSTR_DMA_HT)
7309                         break;
7310
7311                 if (i == RTE_DIM(instr->dma.dst.header_id))
7312                         break;
7313
7314                 if (i && data[i].n_users)
7315                         break;
7316         }
7317
7318         if (i < 2)
7319                 return 0;
7320
7321         *n_pattern_instr = i;
7322         return 1;
7323 }
7324
7325 static void
7326 instr_pattern_dma_many_replace(struct instruction *instr,
7327                                struct instruction_data *data,
7328                                uint32_t n_instr)
7329 {
7330         uint32_t i;
7331
7332         for (i = 1; i < n_instr; i++) {
7333                 instr[0].type++;
7334                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7335                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7336                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7337                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7338
7339                 data[i].invalid = 1;
7340         }
7341 }
7342
7343 static uint32_t
7344 instr_pattern_dma_many_optimize(struct instruction *instructions,
7345                struct instruction_data *instruction_data,
7346                uint32_t n_instructions)
7347 {
7348         uint32_t i;
7349
7350         for (i = 0; i < n_instructions; ) {
7351                 struct instruction *instr = &instructions[i];
7352                 struct instruction_data *data = &instruction_data[i];
7353                 uint32_t n_instr = 0;
7354                 int detected;
7355
7356                 /* DMA many. */
7357                 detected = instr_pattern_dma_many_search(instr,
7358                                                          data,
7359                                                          n_instructions - i,
7360                                                          &n_instr);
7361                 if (detected) {
7362                         instr_pattern_dma_many_replace(instr, data, n_instr);
7363                         i += n_instr;
7364                         continue;
7365                 }
7366
7367                 /* No pattern starting at the current instruction. */
7368                 i++;
7369         }
7370
7371         /* Eliminate the invalid instructions that have been optimized out. */
7372         n_instructions = instr_compact(instructions,
7373                                        instruction_data,
7374                                        n_instructions);
7375
7376         return n_instructions;
7377 }
7378
7379 static uint32_t
7380 instr_optimize(struct rte_swx_pipeline *p,
7381                struct action *a,
7382                struct instruction *instructions,
7383                struct instruction_data *instruction_data,
7384                uint32_t n_instructions)
7385 {
7386         /* Extract many. */
7387         n_instructions = instr_pattern_extract_many_optimize(instructions,
7388                                                              instruction_data,
7389                                                              n_instructions);
7390
7391         /* Emit many + TX. */
7392         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7393                                                              instruction_data,
7394                                                              n_instructions);
7395
7396         /* Mov all + validate. */
7397         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7398                                                                  a,
7399                                                                  instructions,
7400                                                                  instruction_data,
7401                                                                  n_instructions);
7402
7403         /* DMA many. */
7404         n_instructions = instr_pattern_dma_many_optimize(instructions,
7405                                                          instruction_data,
7406                                                          n_instructions);
7407
7408         return n_instructions;
7409 }
7410
7411 static int
7412 instruction_config(struct rte_swx_pipeline *p,
7413                    struct action *a,
7414                    const char **instructions,
7415                    uint32_t n_instructions)
7416 {
7417         struct instruction *instr = NULL;
7418         struct instruction_data *data = NULL;
7419         int err = 0;
7420         uint32_t i;
7421
7422         CHECK(n_instructions, EINVAL);
7423         CHECK(instructions, EINVAL);
7424         for (i = 0; i < n_instructions; i++)
7425                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7426
7427         /* Memory allocation. */
7428         instr = calloc(n_instructions, sizeof(struct instruction));
7429         if (!instr) {
7430                 err = -ENOMEM;
7431                 goto error;
7432         }
7433
7434         data = calloc(n_instructions, sizeof(struct instruction_data));
7435         if (!data) {
7436                 err = -ENOMEM;
7437                 goto error;
7438         }
7439
7440         for (i = 0; i < n_instructions; i++) {
7441                 char *string = strdup(instructions[i]);
7442                 if (!string) {
7443                         err = -ENOMEM;
7444                         goto error;
7445                 }
7446
7447                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7448                 if (err) {
7449                         free(string);
7450                         goto error;
7451                 }
7452
7453                 free(string);
7454         }
7455
7456         err = instr_label_check(data, n_instructions);
7457         if (err)
7458                 goto error;
7459
7460         err = instr_verify(p, a, instr, data, n_instructions);
7461         if (err)
7462                 goto error;
7463
7464         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7465
7466         err = instr_jmp_resolve(instr, data, n_instructions);
7467         if (err)
7468                 goto error;
7469
7470         if (a) {
7471                 a->instructions = instr;
7472                 a->n_instructions = n_instructions;
7473         } else {
7474                 p->instructions = instr;
7475                 p->n_instructions = n_instructions;
7476         }
7477
7478         free(data);
7479         return 0;
7480
7481 error:
7482         free(data);
7483         free(instr);
7484         return err;
7485 }
7486
7487 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7488
7489 static instr_exec_t instruction_table[] = {
7490         [INSTR_RX] = instr_rx_exec,
7491         [INSTR_TX] = instr_tx_exec,
7492         [INSTR_TX_I] = instr_tx_i_exec,
7493
7494         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7495         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7496         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7497         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7498         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7499         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7500         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7501         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7502         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7503         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7504
7505         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7506         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7507         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7508         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7509         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7510         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7511         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7512         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7513         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7514
7515         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7516         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7517
7518         [INSTR_MOV] = instr_mov_exec,
7519         [INSTR_MOV_MH] = instr_mov_mh_exec,
7520         [INSTR_MOV_HM] = instr_mov_hm_exec,
7521         [INSTR_MOV_HH] = instr_mov_hh_exec,
7522         [INSTR_MOV_I] = instr_mov_i_exec,
7523
7524         [INSTR_DMA_HT] = instr_dma_ht_exec,
7525         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7526         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7527         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7528         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7529         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7530         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7531         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7532
7533         [INSTR_ALU_ADD] = instr_alu_add_exec,
7534         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7535         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7536         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7537         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7538         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7539
7540         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7541         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7542         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7543         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7544         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7545         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7546
7547         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7548         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7549         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7550         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7551
7552         [INSTR_ALU_AND] = instr_alu_and_exec,
7553         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7554         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7555         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7556         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7557
7558         [INSTR_ALU_OR] = instr_alu_or_exec,
7559         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7560         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7561         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7562         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7563
7564         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7565         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7566         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7567         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7568         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7569
7570         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7571         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7572         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7573         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7574         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7575         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7576
7577         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7578         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7579         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7580         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7581         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7582         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7583
7584         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7585         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7586         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7587
7588         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7589         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7590         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7591         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7592         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7593         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7594
7595         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7596         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7597         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7598         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7599         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7600         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7601         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7602         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7603         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7604
7605         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7606         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7607         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7608         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7609         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7610         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7611         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7612         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7613         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7614
7615         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7616         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7617         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7618
7619         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7620         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7621         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7622         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7623         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7624         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7625         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7626         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7627         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7628         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7629         [INSTR_METER_IMM] = instr_meter_imm_exec,
7630         [INSTR_METER_IMI] = instr_meter_imi_exec,
7631
7632         [INSTR_TABLE] = instr_table_exec,
7633         [INSTR_SELECTOR] = instr_selector_exec,
7634         [INSTR_LEARNER] = instr_learner_exec,
7635         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7636         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7637         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7638         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7639
7640         [INSTR_JMP] = instr_jmp_exec,
7641         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7642         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7643         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7644         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7645         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7646         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7647
7648         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7649         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7650         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7651         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7652         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7653
7654         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7655         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7656         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7657         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7658         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7659
7660         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7661         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7662         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7663         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7664         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7665         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7666
7667         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7668         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7669         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7670         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7671         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7672         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7673
7674         [INSTR_RETURN] = instr_return_exec,
7675 };
7676
7677 static inline void
7678 instr_exec(struct rte_swx_pipeline *p)
7679 {
7680         struct thread *t = &p->threads[p->thread_id];
7681         struct instruction *ip = t->ip;
7682         instr_exec_t instr = instruction_table[ip->type];
7683
7684         instr(p);
7685 }
7686
7687 /*
7688  * Action.
7689  */
7690 static struct action *
7691 action_find(struct rte_swx_pipeline *p, const char *name)
7692 {
7693         struct action *elem;
7694
7695         if (!name)
7696                 return NULL;
7697
7698         TAILQ_FOREACH(elem, &p->actions, node)
7699                 if (strcmp(elem->name, name) == 0)
7700                         return elem;
7701
7702         return NULL;
7703 }
7704
7705 static struct action *
7706 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7707 {
7708         struct action *action = NULL;
7709
7710         TAILQ_FOREACH(action, &p->actions, node)
7711                 if (action->id == id)
7712                         return action;
7713
7714         return NULL;
7715 }
7716
7717 static struct field *
7718 action_field_find(struct action *a, const char *name)
7719 {
7720         return a->st ? struct_type_field_find(a->st, name) : NULL;
7721 }
7722
7723 static struct field *
7724 action_field_parse(struct action *action, const char *name)
7725 {
7726         if (name[0] != 't' || name[1] != '.')
7727                 return NULL;
7728
7729         return action_field_find(action, &name[2]);
7730 }
7731
7732 static int
7733 action_has_nbo_args(struct action *a)
7734 {
7735         uint32_t i;
7736
7737         /* Return if the action does not have any args. */
7738         if (!a->st)
7739                 return 0; /* FALSE */
7740
7741         for (i = 0; i < a->st->n_fields; i++)
7742                 if (a->args_endianness[i])
7743                         return 1; /* TRUE */
7744
7745         return 0; /* FALSE */
7746 }
7747
7748 static int
7749 action_does_learning(struct action *a)
7750 {
7751         uint32_t i;
7752
7753         for (i = 0; i < a->n_instructions; i++)
7754                 switch (a->instructions[i].type) {
7755                 case INSTR_LEARNER_LEARN:
7756                         return 1; /* TRUE */
7757
7758                 case INSTR_LEARNER_FORGET:
7759                         return 1; /* TRUE */
7760
7761                 default:
7762                         continue;
7763                 }
7764
7765         return 0; /* FALSE */
7766 }
7767
7768 int
7769 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7770                                const char *name,
7771                                const char *args_struct_type_name,
7772                                const char **instructions,
7773                                uint32_t n_instructions)
7774 {
7775         struct struct_type *args_struct_type = NULL;
7776         struct action *a;
7777         int err;
7778
7779         CHECK(p, EINVAL);
7780
7781         CHECK_NAME(name, EINVAL);
7782         CHECK(!action_find(p, name), EEXIST);
7783
7784         if (args_struct_type_name) {
7785                 CHECK_NAME(args_struct_type_name, EINVAL);
7786                 args_struct_type = struct_type_find(p, args_struct_type_name);
7787                 CHECK(args_struct_type, EINVAL);
7788                 CHECK(!args_struct_type->var_size, EINVAL);
7789         }
7790
7791         /* Node allocation. */
7792         a = calloc(1, sizeof(struct action));
7793         CHECK(a, ENOMEM);
7794         if (args_struct_type) {
7795                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7796                 if (!a->args_endianness) {
7797                         free(a);
7798                         CHECK(0, ENOMEM);
7799                 }
7800         }
7801
7802         /* Node initialization. */
7803         strcpy(a->name, name);
7804         a->st = args_struct_type;
7805         a->id = p->n_actions;
7806
7807         /* Instruction translation. */
7808         err = instruction_config(p, a, instructions, n_instructions);
7809         if (err) {
7810                 free(a->args_endianness);
7811                 free(a);
7812                 return err;
7813         }
7814
7815         /* Node add to tailq. */
7816         TAILQ_INSERT_TAIL(&p->actions, a, node);
7817         p->n_actions++;
7818
7819         return 0;
7820 }
7821
7822 static int
7823 action_build(struct rte_swx_pipeline *p)
7824 {
7825         struct action *action;
7826
7827         p->action_instructions = calloc(p->n_actions,
7828                                         sizeof(struct instruction *));
7829         CHECK(p->action_instructions, ENOMEM);
7830
7831         TAILQ_FOREACH(action, &p->actions, node)
7832                 p->action_instructions[action->id] = action->instructions;
7833
7834         return 0;
7835 }
7836
7837 static void
7838 action_build_free(struct rte_swx_pipeline *p)
7839 {
7840         free(p->action_instructions);
7841         p->action_instructions = NULL;
7842 }
7843
7844 static void
7845 action_free(struct rte_swx_pipeline *p)
7846 {
7847         action_build_free(p);
7848
7849         for ( ; ; ) {
7850                 struct action *action;
7851
7852                 action = TAILQ_FIRST(&p->actions);
7853                 if (!action)
7854                         break;
7855
7856                 TAILQ_REMOVE(&p->actions, action, node);
7857                 free(action->instructions);
7858                 free(action);
7859         }
7860 }
7861
7862 static uint32_t
7863 action_arg_src_mov_count(struct action *a,
7864                          uint32_t arg_id,
7865                          struct instruction *instructions,
7866                          struct instruction_data *instruction_data,
7867                          uint32_t n_instructions)
7868 {
7869         uint32_t offset, n_users = 0, i;
7870
7871         if (!a->st ||
7872             (arg_id >= a->st->n_fields) ||
7873             !instructions ||
7874             !instruction_data ||
7875             !n_instructions)
7876                 return 0;
7877
7878         offset = a->st->fields[arg_id].offset / 8;
7879
7880         for (i = 0; i < n_instructions; i++) {
7881                 struct instruction *instr = &instructions[i];
7882                 struct instruction_data *data = &instruction_data[i];
7883
7884                 if (data->invalid ||
7885                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7886                     instr->mov.src.struct_id ||
7887                     (instr->mov.src.offset != offset))
7888                         continue;
7889
7890                 n_users++;
7891         }
7892
7893         return n_users;
7894 }
7895
7896 /*
7897  * Table.
7898  */
7899 static struct table_type *
7900 table_type_find(struct rte_swx_pipeline *p, const char *name)
7901 {
7902         struct table_type *elem;
7903
7904         TAILQ_FOREACH(elem, &p->table_types, node)
7905                 if (strcmp(elem->name, name) == 0)
7906                         return elem;
7907
7908         return NULL;
7909 }
7910
7911 static struct table_type *
7912 table_type_resolve(struct rte_swx_pipeline *p,
7913                    const char *recommended_type_name,
7914                    enum rte_swx_table_match_type match_type)
7915 {
7916         struct table_type *elem;
7917
7918         /* Only consider the recommended type if the match type is correct. */
7919         if (recommended_type_name)
7920                 TAILQ_FOREACH(elem, &p->table_types, node)
7921                         if (!strcmp(elem->name, recommended_type_name) &&
7922                             (elem->match_type == match_type))
7923                                 return elem;
7924
7925         /* Ignore the recommended type and get the first element with this match
7926          * type.
7927          */
7928         TAILQ_FOREACH(elem, &p->table_types, node)
7929                 if (elem->match_type == match_type)
7930                         return elem;
7931
7932         return NULL;
7933 }
7934
7935 static struct table *
7936 table_find(struct rte_swx_pipeline *p, const char *name)
7937 {
7938         struct table *elem;
7939
7940         TAILQ_FOREACH(elem, &p->tables, node)
7941                 if (strcmp(elem->name, name) == 0)
7942                         return elem;
7943
7944         return NULL;
7945 }
7946
7947 static struct table *
7948 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7949 {
7950         struct table *table = NULL;
7951
7952         TAILQ_FOREACH(table, &p->tables, node)
7953                 if (table->id == id)
7954                         return table;
7955
7956         return NULL;
7957 }
7958
7959 int
7960 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7961                                      const char *name,
7962                                      enum rte_swx_table_match_type match_type,
7963                                      struct rte_swx_table_ops *ops)
7964 {
7965         struct table_type *elem;
7966
7967         CHECK(p, EINVAL);
7968
7969         CHECK_NAME(name, EINVAL);
7970         CHECK(!table_type_find(p, name), EEXIST);
7971
7972         CHECK(ops, EINVAL);
7973         CHECK(ops->create, EINVAL);
7974         CHECK(ops->lkp, EINVAL);
7975         CHECK(ops->free, EINVAL);
7976
7977         /* Node allocation. */
7978         elem = calloc(1, sizeof(struct table_type));
7979         CHECK(elem, ENOMEM);
7980
7981         /* Node initialization. */
7982         strcpy(elem->name, name);
7983         elem->match_type = match_type;
7984         memcpy(&elem->ops, ops, sizeof(*ops));
7985
7986         /* Node add to tailq. */
7987         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7988
7989         return 0;
7990 }
7991
7992 static int
7993 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7994                          uint32_t n_fields,
7995                          enum rte_swx_table_match_type *match_type)
7996 {
7997         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7998
7999         for (i = 0; i < n_fields; i++) {
8000                 struct rte_swx_match_field_params  *f = &fields[i];
8001
8002                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
8003                         n_fields_em++;
8004
8005                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
8006                         n_fields_lpm++;
8007         }
8008
8009         if ((n_fields_lpm > 1) ||
8010             (n_fields_lpm && (n_fields_em != n_fields - 1)))
8011                 return -EINVAL;
8012
8013         *match_type = (n_fields_em == n_fields) ?
8014                        RTE_SWX_TABLE_MATCH_EXACT :
8015                        RTE_SWX_TABLE_MATCH_WILDCARD;
8016
8017         return 0;
8018 }
8019
8020 static int
8021 table_match_fields_check(struct rte_swx_pipeline *p,
8022                          struct rte_swx_pipeline_table_params *params,
8023                          struct header **header)
8024 {
8025         struct header *h0 = NULL;
8026         struct field *hf, *mf;
8027         uint32_t *offset = NULL, i;
8028         int status = 0;
8029
8030         /* Return if no match fields. */
8031         if (!params->n_fields) {
8032                 if (params->fields) {
8033                         status = -EINVAL;
8034                         goto end;
8035                 }
8036
8037                 if (header)
8038                         *header = NULL;
8039
8040                 return 0;
8041         }
8042
8043         /* Memory allocation. */
8044         offset = calloc(params->n_fields, sizeof(uint32_t));
8045         if (!offset) {
8046                 status = -ENOMEM;
8047                 goto end;
8048         }
8049
8050         /* Check that all the match fields belong to either the same header or
8051          * to the meta-data.
8052          */
8053         hf = header_field_parse(p, params->fields[0].name, &h0);
8054         mf = metadata_field_parse(p, params->fields[0].name);
8055         if ((!hf && !mf) || (hf && hf->var_size)) {
8056                 status = -EINVAL;
8057                 goto end;
8058         }
8059
8060         offset[0] = h0 ? hf->offset : mf->offset;
8061
8062         for (i = 1; i < params->n_fields; i++)
8063                 if (h0) {
8064                         struct header *h;
8065
8066                         hf = header_field_parse(p, params->fields[i].name, &h);
8067                         if (!hf || (h->id != h0->id) || hf->var_size) {
8068                                 status = -EINVAL;
8069                                 goto end;
8070                         }
8071
8072                         offset[i] = hf->offset;
8073                 } else {
8074                         mf = metadata_field_parse(p, params->fields[i].name);
8075                         if (!mf) {
8076                                 status = -EINVAL;
8077                                 goto end;
8078                         }
8079
8080                         offset[i] = mf->offset;
8081                 }
8082
8083         /* Check that there are no duplicated match fields. */
8084         for (i = 0; i < params->n_fields; i++) {
8085                 uint32_t j;
8086
8087                 for (j = 0; j < i; j++)
8088                         if (offset[j] == offset[i]) {
8089                                 status = -EINVAL;
8090                                 goto end;
8091                         }
8092         }
8093
8094         /* Return. */
8095         if (header)
8096                 *header = h0;
8097
8098 end:
8099         free(offset);
8100         return status;
8101 }
8102
8103 int
8104 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8105                               const char *name,
8106                               struct rte_swx_pipeline_table_params *params,
8107                               const char *recommended_table_type_name,
8108                               const char *args,
8109                               uint32_t size)
8110 {
8111         struct table_type *type;
8112         struct table *t;
8113         struct action *default_action;
8114         struct header *header = NULL;
8115         uint32_t action_data_size_max = 0, i;
8116         int status = 0;
8117
8118         CHECK(p, EINVAL);
8119
8120         CHECK_NAME(name, EINVAL);
8121         CHECK(!table_find(p, name), EEXIST);
8122         CHECK(!selector_find(p, name), EEXIST);
8123         CHECK(!learner_find(p, name), EEXIST);
8124
8125         CHECK(params, EINVAL);
8126
8127         /* Match checks. */
8128         status = table_match_fields_check(p, params, &header);
8129         if (status)
8130                 return status;
8131
8132         /* Action checks. */
8133         CHECK(params->n_actions, EINVAL);
8134         CHECK(params->action_names, EINVAL);
8135         for (i = 0; i < params->n_actions; i++) {
8136                 const char *action_name = params->action_names[i];
8137                 struct action *a;
8138                 uint32_t action_data_size;
8139
8140                 CHECK_NAME(action_name, EINVAL);
8141
8142                 a = action_find(p, action_name);
8143                 CHECK(a, EINVAL);
8144                 CHECK(!action_does_learning(a), EINVAL);
8145
8146                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8147                 if (action_data_size > action_data_size_max)
8148                         action_data_size_max = action_data_size;
8149         }
8150
8151         CHECK_NAME(params->default_action_name, EINVAL);
8152         for (i = 0; i < p->n_actions; i++)
8153                 if (!strcmp(params->action_names[i],
8154                             params->default_action_name))
8155                         break;
8156         CHECK(i < params->n_actions, EINVAL);
8157         default_action = action_find(p, params->default_action_name);
8158         CHECK((default_action->st && params->default_action_data) ||
8159               !params->default_action_data, EINVAL);
8160
8161         /* Table type checks. */
8162         if (recommended_table_type_name)
8163                 CHECK_NAME(recommended_table_type_name, EINVAL);
8164
8165         if (params->n_fields) {
8166                 enum rte_swx_table_match_type match_type;
8167
8168                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8169                 if (status)
8170                         return status;
8171
8172                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8173                 CHECK(type, EINVAL);
8174         } else {
8175                 type = NULL;
8176         }
8177
8178         /* Memory allocation. */
8179         t = calloc(1, sizeof(struct table));
8180         CHECK(t, ENOMEM);
8181
8182         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8183         if (!t->fields) {
8184                 free(t);
8185                 CHECK(0, ENOMEM);
8186         }
8187
8188         t->actions = calloc(params->n_actions, sizeof(struct action *));
8189         if (!t->actions) {
8190                 free(t->fields);
8191                 free(t);
8192                 CHECK(0, ENOMEM);
8193         }
8194
8195         if (action_data_size_max) {
8196                 t->default_action_data = calloc(1, action_data_size_max);
8197                 if (!t->default_action_data) {
8198                         free(t->actions);
8199                         free(t->fields);
8200                         free(t);
8201                         CHECK(0, ENOMEM);
8202                 }
8203         }
8204
8205         /* Node initialization. */
8206         strcpy(t->name, name);
8207         if (args && args[0])
8208                 strcpy(t->args, args);
8209         t->type = type;
8210
8211         for (i = 0; i < params->n_fields; i++) {
8212                 struct rte_swx_match_field_params *field = &params->fields[i];
8213                 struct match_field *f = &t->fields[i];
8214
8215                 f->match_type = field->match_type;
8216                 f->field = header ?
8217                         header_field_parse(p, field->name, NULL) :
8218                         metadata_field_parse(p, field->name);
8219         }
8220         t->n_fields = params->n_fields;
8221         t->header = header;
8222
8223         for (i = 0; i < params->n_actions; i++)
8224                 t->actions[i] = action_find(p, params->action_names[i]);
8225         t->default_action = default_action;
8226         if (default_action->st)
8227                 memcpy(t->default_action_data,
8228                        params->default_action_data,
8229                        default_action->st->n_bits / 8);
8230         t->n_actions = params->n_actions;
8231         t->default_action_is_const = params->default_action_is_const;
8232         t->action_data_size_max = action_data_size_max;
8233
8234         t->size = size;
8235         t->id = p->n_tables;
8236
8237         /* Node add to tailq. */
8238         TAILQ_INSERT_TAIL(&p->tables, t, node);
8239         p->n_tables++;
8240
8241         return 0;
8242 }
8243
8244 static struct rte_swx_table_params *
8245 table_params_get(struct table *table)
8246 {
8247         struct rte_swx_table_params *params;
8248         struct field *first, *last;
8249         uint8_t *key_mask;
8250         uint32_t key_size, key_offset, action_data_size, i;
8251
8252         /* Memory allocation. */
8253         params = calloc(1, sizeof(struct rte_swx_table_params));
8254         if (!params)
8255                 return NULL;
8256
8257         /* Find first (smallest offset) and last (biggest offset) match fields. */
8258         first = table->fields[0].field;
8259         last = table->fields[0].field;
8260
8261         for (i = 0; i < table->n_fields; i++) {
8262                 struct field *f = table->fields[i].field;
8263
8264                 if (f->offset < first->offset)
8265                         first = f;
8266
8267                 if (f->offset > last->offset)
8268                         last = f;
8269         }
8270
8271         /* Key offset and size. */
8272         key_offset = first->offset / 8;
8273         key_size = (last->offset + last->n_bits - first->offset) / 8;
8274
8275         /* Memory allocation. */
8276         key_mask = calloc(1, key_size);
8277         if (!key_mask) {
8278                 free(params);
8279                 return NULL;
8280         }
8281
8282         /* Key mask. */
8283         for (i = 0; i < table->n_fields; i++) {
8284                 struct field *f = table->fields[i].field;
8285                 uint32_t start = (f->offset - first->offset) / 8;
8286                 size_t size = f->n_bits / 8;
8287
8288                 memset(&key_mask[start], 0xFF, size);
8289         }
8290
8291         /* Action data size. */
8292         action_data_size = 0;
8293         for (i = 0; i < table->n_actions; i++) {
8294                 struct action *action = table->actions[i];
8295                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8296
8297                 if (ads > action_data_size)
8298                         action_data_size = ads;
8299         }
8300
8301         /* Fill in. */
8302         params->match_type = table->type->match_type;
8303         params->key_size = key_size;
8304         params->key_offset = key_offset;
8305         params->key_mask0 = key_mask;
8306         params->action_data_size = action_data_size;
8307         params->n_keys_max = table->size;
8308
8309         return params;
8310 }
8311
8312 static void
8313 table_params_free(struct rte_swx_table_params *params)
8314 {
8315         if (!params)
8316                 return;
8317
8318         free(params->key_mask0);
8319         free(params);
8320 }
8321
8322 static int
8323 table_stub_lkp(void *table __rte_unused,
8324                void *mailbox __rte_unused,
8325                uint8_t **key __rte_unused,
8326                uint64_t *action_id __rte_unused,
8327                uint8_t **action_data __rte_unused,
8328                int *hit)
8329 {
8330         *hit = 0;
8331         return 1; /* DONE. */
8332 }
8333
8334 static int
8335 table_build(struct rte_swx_pipeline *p)
8336 {
8337         uint32_t i;
8338
8339         /* Per pipeline: table statistics. */
8340         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8341         CHECK(p->table_stats, ENOMEM);
8342
8343         for (i = 0; i < p->n_tables; i++) {
8344                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8345                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8346         }
8347
8348         /* Per thread: table runt-time. */
8349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8350                 struct thread *t = &p->threads[i];
8351                 struct table *table;
8352
8353                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8354                 CHECK(t->tables, ENOMEM);
8355
8356                 TAILQ_FOREACH(table, &p->tables, node) {
8357                         struct table_runtime *r = &t->tables[table->id];
8358
8359                         if (table->type) {
8360                                 uint64_t size;
8361
8362                                 size = table->type->ops.mailbox_size_get();
8363
8364                                 /* r->func. */
8365                                 r->func = table->type->ops.lkp;
8366
8367                                 /* r->mailbox. */
8368                                 if (size) {
8369                                         r->mailbox = calloc(1, size);
8370                                         CHECK(r->mailbox, ENOMEM);
8371                                 }
8372
8373                                 /* r->key. */
8374                                 r->key = table->header ?
8375                                         &t->structs[table->header->struct_id] :
8376                                         &t->structs[p->metadata_struct_id];
8377                         } else {
8378                                 r->func = table_stub_lkp;
8379                         }
8380                 }
8381         }
8382
8383         return 0;
8384 }
8385
8386 static void
8387 table_build_free(struct rte_swx_pipeline *p)
8388 {
8389         uint32_t i;
8390
8391         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8392                 struct thread *t = &p->threads[i];
8393                 uint32_t j;
8394
8395                 if (!t->tables)
8396                         continue;
8397
8398                 for (j = 0; j < p->n_tables; j++) {
8399                         struct table_runtime *r = &t->tables[j];
8400
8401                         free(r->mailbox);
8402                 }
8403
8404                 free(t->tables);
8405                 t->tables = NULL;
8406         }
8407
8408         if (p->table_stats) {
8409                 for (i = 0; i < p->n_tables; i++)
8410                         free(p->table_stats[i].n_pkts_action);
8411
8412                 free(p->table_stats);
8413         }
8414 }
8415
8416 static void
8417 table_free(struct rte_swx_pipeline *p)
8418 {
8419         table_build_free(p);
8420
8421         /* Tables. */
8422         for ( ; ; ) {
8423                 struct table *elem;
8424
8425                 elem = TAILQ_FIRST(&p->tables);
8426                 if (!elem)
8427                         break;
8428
8429                 TAILQ_REMOVE(&p->tables, elem, node);
8430                 free(elem->fields);
8431                 free(elem->actions);
8432                 free(elem->default_action_data);
8433                 free(elem);
8434         }
8435
8436         /* Table types. */
8437         for ( ; ; ) {
8438                 struct table_type *elem;
8439
8440                 elem = TAILQ_FIRST(&p->table_types);
8441                 if (!elem)
8442                         break;
8443
8444                 TAILQ_REMOVE(&p->table_types, elem, node);
8445                 free(elem);
8446         }
8447 }
8448
8449 /*
8450  * Selector.
8451  */
8452 static struct selector *
8453 selector_find(struct rte_swx_pipeline *p, const char *name)
8454 {
8455         struct selector *s;
8456
8457         TAILQ_FOREACH(s, &p->selectors, node)
8458                 if (strcmp(s->name, name) == 0)
8459                         return s;
8460
8461         return NULL;
8462 }
8463
8464 static struct selector *
8465 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8466 {
8467         struct selector *s = NULL;
8468
8469         TAILQ_FOREACH(s, &p->selectors, node)
8470                 if (s->id == id)
8471                         return s;
8472
8473         return NULL;
8474 }
8475
8476 static int
8477 selector_fields_check(struct rte_swx_pipeline *p,
8478                       struct rte_swx_pipeline_selector_params *params,
8479                       struct header **header)
8480 {
8481         struct header *h0 = NULL;
8482         struct field *hf, *mf;
8483         uint32_t i;
8484
8485         /* Return if no selector fields. */
8486         if (!params->n_selector_fields || !params->selector_field_names)
8487                 return -EINVAL;
8488
8489         /* Check that all the selector fields either belong to the same header
8490          * or are all meta-data fields.
8491          */
8492         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8493         mf = metadata_field_parse(p, params->selector_field_names[0]);
8494         if (!hf && !mf)
8495                 return -EINVAL;
8496
8497         for (i = 1; i < params->n_selector_fields; i++)
8498                 if (h0) {
8499                         struct header *h;
8500
8501                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8502                         if (!hf || (h->id != h0->id))
8503                                 return -EINVAL;
8504                 } else {
8505                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8506                         if (!mf)
8507                                 return -EINVAL;
8508                 }
8509
8510         /* Check that there are no duplicated match fields. */
8511         for (i = 0; i < params->n_selector_fields; i++) {
8512                 const char *field_name = params->selector_field_names[i];
8513                 uint32_t j;
8514
8515                 for (j = i + 1; j < params->n_selector_fields; j++)
8516                         if (!strcmp(params->selector_field_names[j], field_name))
8517                                 return -EINVAL;
8518         }
8519
8520         /* Return. */
8521         if (header)
8522                 *header = h0;
8523
8524         return 0;
8525 }
8526
8527 int
8528 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8529                                  const char *name,
8530                                  struct rte_swx_pipeline_selector_params *params)
8531 {
8532         struct selector *s;
8533         struct header *selector_header = NULL;
8534         struct field *group_id_field, *member_id_field;
8535         uint32_t i;
8536         int status = 0;
8537
8538         CHECK(p, EINVAL);
8539
8540         CHECK_NAME(name, EINVAL);
8541         CHECK(!table_find(p, name), EEXIST);
8542         CHECK(!selector_find(p, name), EEXIST);
8543         CHECK(!learner_find(p, name), EEXIST);
8544
8545         CHECK(params, EINVAL);
8546
8547         CHECK_NAME(params->group_id_field_name, EINVAL);
8548         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8549         CHECK(group_id_field, EINVAL);
8550
8551         for (i = 0; i < params->n_selector_fields; i++) {
8552                 const char *field_name = params->selector_field_names[i];
8553
8554                 CHECK_NAME(field_name, EINVAL);
8555         }
8556         status = selector_fields_check(p, params, &selector_header);
8557         if (status)
8558                 return status;
8559
8560         CHECK_NAME(params->member_id_field_name, EINVAL);
8561         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8562         CHECK(member_id_field, EINVAL);
8563
8564         CHECK(params->n_groups_max, EINVAL);
8565
8566         CHECK(params->n_members_per_group_max, EINVAL);
8567
8568         /* Memory allocation. */
8569         s = calloc(1, sizeof(struct selector));
8570         if (!s) {
8571                 status = -ENOMEM;
8572                 goto error;
8573         }
8574
8575         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8576         if (!s->selector_fields) {
8577                 status = -ENOMEM;
8578                 goto error;
8579         }
8580
8581         /* Node initialization. */
8582         strcpy(s->name, name);
8583
8584         s->group_id_field = group_id_field;
8585
8586         for (i = 0; i < params->n_selector_fields; i++) {
8587                 const char *field_name = params->selector_field_names[i];
8588
8589                 s->selector_fields[i] = selector_header ?
8590                         header_field_parse(p, field_name, NULL) :
8591                         metadata_field_parse(p, field_name);
8592         }
8593
8594         s->n_selector_fields = params->n_selector_fields;
8595
8596         s->selector_header = selector_header;
8597
8598         s->member_id_field = member_id_field;
8599
8600         s->n_groups_max = params->n_groups_max;
8601
8602         s->n_members_per_group_max = params->n_members_per_group_max;
8603
8604         s->id = p->n_selectors;
8605
8606         /* Node add to tailq. */
8607         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8608         p->n_selectors++;
8609
8610         return 0;
8611
8612 error:
8613         if (!s)
8614                 return status;
8615
8616         free(s->selector_fields);
8617
8618         free(s);
8619
8620         return status;
8621 }
8622
8623 static void
8624 selector_params_free(struct rte_swx_table_selector_params *params)
8625 {
8626         if (!params)
8627                 return;
8628
8629         free(params->selector_mask);
8630
8631         free(params);
8632 }
8633
8634 static struct rte_swx_table_selector_params *
8635 selector_table_params_get(struct selector *s)
8636 {
8637         struct rte_swx_table_selector_params *params = NULL;
8638         struct field *first, *last;
8639         uint32_t i;
8640
8641         /* Memory allocation. */
8642         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8643         if (!params)
8644                 goto error;
8645
8646         /* Group ID. */
8647         params->group_id_offset = s->group_id_field->offset / 8;
8648
8649         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8650         first = s->selector_fields[0];
8651         last = s->selector_fields[0];
8652
8653         for (i = 0; i < s->n_selector_fields; i++) {
8654                 struct field *f = s->selector_fields[i];
8655
8656                 if (f->offset < first->offset)
8657                         first = f;
8658
8659                 if (f->offset > last->offset)
8660                         last = f;
8661         }
8662
8663         /* Selector offset and size. */
8664         params->selector_offset = first->offset / 8;
8665         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8666
8667         /* Memory allocation. */
8668         params->selector_mask = calloc(1, params->selector_size);
8669         if (!params->selector_mask)
8670                 goto error;
8671
8672         /* Selector mask. */
8673         for (i = 0; i < s->n_selector_fields; i++) {
8674                 struct field *f = s->selector_fields[i];
8675                 uint32_t start = (f->offset - first->offset) / 8;
8676                 size_t size = f->n_bits / 8;
8677
8678                 memset(&params->selector_mask[start], 0xFF, size);
8679         }
8680
8681         /* Member ID. */
8682         params->member_id_offset = s->member_id_field->offset / 8;
8683
8684         /* Maximum number of groups. */
8685         params->n_groups_max = s->n_groups_max;
8686
8687         /* Maximum number of members per group. */
8688         params->n_members_per_group_max = s->n_members_per_group_max;
8689
8690         return params;
8691
8692 error:
8693         selector_params_free(params);
8694         return NULL;
8695 }
8696
8697 static void
8698 selector_build_free(struct rte_swx_pipeline *p)
8699 {
8700         uint32_t i;
8701
8702         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8703                 struct thread *t = &p->threads[i];
8704                 uint32_t j;
8705
8706                 if (!t->selectors)
8707                         continue;
8708
8709                 for (j = 0; j < p->n_selectors; j++) {
8710                         struct selector_runtime *r = &t->selectors[j];
8711
8712                         free(r->mailbox);
8713                 }
8714
8715                 free(t->selectors);
8716                 t->selectors = NULL;
8717         }
8718
8719         free(p->selector_stats);
8720         p->selector_stats = NULL;
8721 }
8722
8723 static int
8724 selector_build(struct rte_swx_pipeline *p)
8725 {
8726         uint32_t i;
8727         int status = 0;
8728
8729         /* Per pipeline: selector statistics. */
8730         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8731         if (!p->selector_stats) {
8732                 status = -ENOMEM;
8733                 goto error;
8734         }
8735
8736         /* Per thread: selector run-time. */
8737         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8738                 struct thread *t = &p->threads[i];
8739                 struct selector *s;
8740
8741                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8742                 if (!t->selectors) {
8743                         status = -ENOMEM;
8744                         goto error;
8745                 }
8746
8747                 TAILQ_FOREACH(s, &p->selectors, node) {
8748                         struct selector_runtime *r = &t->selectors[s->id];
8749                         uint64_t size;
8750
8751                         /* r->mailbox. */
8752                         size = rte_swx_table_selector_mailbox_size_get();
8753                         if (size) {
8754                                 r->mailbox = calloc(1, size);
8755                                 if (!r->mailbox) {
8756                                         status = -ENOMEM;
8757                                         goto error;
8758                                 }
8759                         }
8760
8761                         /* r->group_id_buffer. */
8762                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8763
8764                         /* r->selector_buffer. */
8765                         r->selector_buffer = s->selector_header ?
8766                                 &t->structs[s->selector_header->struct_id] :
8767                                 &t->structs[p->metadata_struct_id];
8768
8769                         /* r->member_id_buffer. */
8770                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8771                 }
8772         }
8773
8774         return 0;
8775
8776 error:
8777         selector_build_free(p);
8778         return status;
8779 }
8780
8781 static void
8782 selector_free(struct rte_swx_pipeline *p)
8783 {
8784         selector_build_free(p);
8785
8786         /* Selector tables. */
8787         for ( ; ; ) {
8788                 struct selector *elem;
8789
8790                 elem = TAILQ_FIRST(&p->selectors);
8791                 if (!elem)
8792                         break;
8793
8794                 TAILQ_REMOVE(&p->selectors, elem, node);
8795                 free(elem->selector_fields);
8796                 free(elem);
8797         }
8798 }
8799
8800 /*
8801  * Learner table.
8802  */
8803 static struct learner *
8804 learner_find(struct rte_swx_pipeline *p, const char *name)
8805 {
8806         struct learner *l;
8807
8808         TAILQ_FOREACH(l, &p->learners, node)
8809                 if (!strcmp(l->name, name))
8810                         return l;
8811
8812         return NULL;
8813 }
8814
8815 static struct learner *
8816 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8817 {
8818         struct learner *l = NULL;
8819
8820         TAILQ_FOREACH(l, &p->learners, node)
8821                 if (l->id == id)
8822                         return l;
8823
8824         return NULL;
8825 }
8826
8827 static int
8828 learner_match_fields_check(struct rte_swx_pipeline *p,
8829                            struct rte_swx_pipeline_learner_params *params,
8830                            struct header **header)
8831 {
8832         struct header *h0 = NULL;
8833         struct field *hf, *mf;
8834         uint32_t i;
8835
8836         /* Return if no match fields. */
8837         if (!params->n_fields || !params->field_names)
8838                 return -EINVAL;
8839
8840         /* Check that all the match fields either belong to the same header
8841          * or are all meta-data fields.
8842          */
8843         hf = header_field_parse(p, params->field_names[0], &h0);
8844         mf = metadata_field_parse(p, params->field_names[0]);
8845         if (!hf && !mf)
8846                 return -EINVAL;
8847
8848         for (i = 1; i < params->n_fields; i++)
8849                 if (h0) {
8850                         struct header *h;
8851
8852                         hf = header_field_parse(p, params->field_names[i], &h);
8853                         if (!hf || (h->id != h0->id))
8854                                 return -EINVAL;
8855                 } else {
8856                         mf = metadata_field_parse(p, params->field_names[i]);
8857                         if (!mf)
8858                                 return -EINVAL;
8859                 }
8860
8861         /* Check that there are no duplicated match fields. */
8862         for (i = 0; i < params->n_fields; i++) {
8863                 const char *field_name = params->field_names[i];
8864                 uint32_t j;
8865
8866                 for (j = i + 1; j < params->n_fields; j++)
8867                         if (!strcmp(params->field_names[j], field_name))
8868                                 return -EINVAL;
8869         }
8870
8871         /* Return. */
8872         if (header)
8873                 *header = h0;
8874
8875         return 0;
8876 }
8877
8878 static int
8879 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8880 {
8881         struct struct_type *mst = p->metadata_st, *ast = a->st;
8882         struct field *mf, *af;
8883         uint32_t mf_pos, i;
8884
8885         if (!ast) {
8886                 if (mf_name)
8887                         return -EINVAL;
8888
8889                 return 0;
8890         }
8891
8892         /* Check that mf_name is the name of a valid meta-data field. */
8893         CHECK_NAME(mf_name, EINVAL);
8894         mf = metadata_field_parse(p, mf_name);
8895         CHECK(mf, EINVAL);
8896
8897         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8898          * all the action arguments.
8899          */
8900         mf_pos = mf - mst->fields;
8901         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8902
8903         /* Check that the size of each of the identified meta-data fields matches exactly the size
8904          * of the corresponding action argument.
8905          */
8906         for (i = 0; i < ast->n_fields; i++) {
8907                 mf = &mst->fields[mf_pos + i];
8908                 af = &ast->fields[i];
8909
8910                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8911         }
8912
8913         return 0;
8914 }
8915
8916 static int
8917 learner_action_learning_check(struct rte_swx_pipeline *p,
8918                               struct action *action,
8919                               const char **action_names,
8920                               uint32_t n_actions)
8921 {
8922         uint32_t i;
8923
8924         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8925          * the action passed as argument to the "learn" instruction) is also enabled for the
8926          * current learner table.
8927          */
8928         for (i = 0; i < action->n_instructions; i++) {
8929                 struct instruction *instr = &action->instructions[i];
8930                 uint32_t found = 0, j;
8931
8932                 if (instr->type != INSTR_LEARNER_LEARN)
8933                         continue;
8934
8935                 for (j = 0; j < n_actions; j++) {
8936                         struct action *a;
8937
8938                         a = action_find(p, action_names[j]);
8939                         if (!a)
8940                                 return -EINVAL;
8941
8942                         if (a->id == instr->learn.action_id)
8943                                 found = 1;
8944                 }
8945
8946                 if (!found)
8947                         return -EINVAL;
8948         }
8949
8950         return 0;
8951 }
8952
8953 int
8954 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8955                               const char *name,
8956                               struct rte_swx_pipeline_learner_params *params,
8957                               uint32_t size,
8958                               uint32_t timeout)
8959 {
8960         struct learner *l = NULL;
8961         struct action *default_action;
8962         struct header *header = NULL;
8963         uint32_t action_data_size_max = 0, i;
8964         int status = 0;
8965
8966         CHECK(p, EINVAL);
8967
8968         CHECK_NAME(name, EINVAL);
8969         CHECK(!table_find(p, name), EEXIST);
8970         CHECK(!selector_find(p, name), EEXIST);
8971         CHECK(!learner_find(p, name), EEXIST);
8972
8973         CHECK(params, EINVAL);
8974
8975         /* Match checks. */
8976         status = learner_match_fields_check(p, params, &header);
8977         if (status)
8978                 return status;
8979
8980         /* Action checks. */
8981         CHECK(params->n_actions, EINVAL);
8982
8983         CHECK(params->action_names, EINVAL);
8984         for (i = 0; i < params->n_actions; i++) {
8985                 const char *action_name = params->action_names[i];
8986                 const char *action_field_name = params->action_field_names[i];
8987                 struct action *a;
8988                 uint32_t action_data_size;
8989
8990                 CHECK_NAME(action_name, EINVAL);
8991
8992                 a = action_find(p, action_name);
8993                 CHECK(a, EINVAL);
8994
8995                 status = learner_action_args_check(p, a, action_field_name);
8996                 if (status)
8997                         return status;
8998
8999                 status = learner_action_learning_check(p,
9000                                                        a,
9001                                                        params->action_names,
9002                                                        params->n_actions);
9003                 if (status)
9004                         return status;
9005
9006                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9007                 if (action_data_size > action_data_size_max)
9008                         action_data_size_max = action_data_size;
9009         }
9010
9011         CHECK_NAME(params->default_action_name, EINVAL);
9012         for (i = 0; i < p->n_actions; i++)
9013                 if (!strcmp(params->action_names[i],
9014                             params->default_action_name))
9015                         break;
9016         CHECK(i < params->n_actions, EINVAL);
9017
9018         default_action = action_find(p, params->default_action_name);
9019         CHECK((default_action->st && params->default_action_data) ||
9020               !params->default_action_data, EINVAL);
9021
9022         /* Any other checks. */
9023         CHECK(size, EINVAL);
9024         CHECK(timeout, EINVAL);
9025
9026         /* Memory allocation. */
9027         l = calloc(1, sizeof(struct learner));
9028         if (!l)
9029                 goto nomem;
9030
9031         l->fields = calloc(params->n_fields, sizeof(struct field *));
9032         if (!l->fields)
9033                 goto nomem;
9034
9035         l->actions = calloc(params->n_actions, sizeof(struct action *));
9036         if (!l->actions)
9037                 goto nomem;
9038
9039         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9040         if (!l->action_arg)
9041                 goto nomem;
9042
9043         if (action_data_size_max) {
9044                 l->default_action_data = calloc(1, action_data_size_max);
9045                 if (!l->default_action_data)
9046                         goto nomem;
9047         }
9048
9049         /* Node initialization. */
9050         strcpy(l->name, name);
9051
9052         for (i = 0; i < params->n_fields; i++) {
9053                 const char *field_name = params->field_names[i];
9054
9055                 l->fields[i] = header ?
9056                         header_field_parse(p, field_name, NULL) :
9057                         metadata_field_parse(p, field_name);
9058         }
9059
9060         l->n_fields = params->n_fields;
9061
9062         l->header = header;
9063
9064         for (i = 0; i < params->n_actions; i++) {
9065                 const char *mf_name = params->action_field_names[i];
9066
9067                 l->actions[i] = action_find(p, params->action_names[i]);
9068
9069                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9070         }
9071
9072         l->default_action = default_action;
9073
9074         if (default_action->st)
9075                 memcpy(l->default_action_data,
9076                        params->default_action_data,
9077                        default_action->st->n_bits / 8);
9078
9079         l->n_actions = params->n_actions;
9080
9081         l->default_action_is_const = params->default_action_is_const;
9082
9083         l->action_data_size_max = action_data_size_max;
9084
9085         l->size = size;
9086
9087         l->timeout = timeout;
9088
9089         l->id = p->n_learners;
9090
9091         /* Node add to tailq. */
9092         TAILQ_INSERT_TAIL(&p->learners, l, node);
9093         p->n_learners++;
9094
9095         return 0;
9096
9097 nomem:
9098         if (!l)
9099                 return -ENOMEM;
9100
9101         free(l->action_arg);
9102         free(l->actions);
9103         free(l->fields);
9104         free(l);
9105
9106         return -ENOMEM;
9107 }
9108
9109 static void
9110 learner_params_free(struct rte_swx_table_learner_params *params)
9111 {
9112         if (!params)
9113                 return;
9114
9115         free(params->key_mask0);
9116
9117         free(params);
9118 }
9119
9120 static struct rte_swx_table_learner_params *
9121 learner_params_get(struct learner *l)
9122 {
9123         struct rte_swx_table_learner_params *params = NULL;
9124         struct field *first, *last;
9125         uint32_t i;
9126
9127         /* Memory allocation. */
9128         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9129         if (!params)
9130                 goto error;
9131
9132         /* Find first (smallest offset) and last (biggest offset) match fields. */
9133         first = l->fields[0];
9134         last = l->fields[0];
9135
9136         for (i = 0; i < l->n_fields; i++) {
9137                 struct field *f = l->fields[i];
9138
9139                 if (f->offset < first->offset)
9140                         first = f;
9141
9142                 if (f->offset > last->offset)
9143                         last = f;
9144         }
9145
9146         /* Key offset and size. */
9147         params->key_offset = first->offset / 8;
9148         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9149
9150         /* Memory allocation. */
9151         params->key_mask0 = calloc(1, params->key_size);
9152         if (!params->key_mask0)
9153                 goto error;
9154
9155         /* Key mask. */
9156         for (i = 0; i < l->n_fields; i++) {
9157                 struct field *f = l->fields[i];
9158                 uint32_t start = (f->offset - first->offset) / 8;
9159                 size_t size = f->n_bits / 8;
9160
9161                 memset(&params->key_mask0[start], 0xFF, size);
9162         }
9163
9164         /* Action data size. */
9165         params->action_data_size = l->action_data_size_max;
9166
9167         /* Maximum number of keys. */
9168         params->n_keys_max = l->size;
9169
9170         /* Timeout. */
9171         params->key_timeout = l->timeout;
9172
9173         return params;
9174
9175 error:
9176         learner_params_free(params);
9177         return NULL;
9178 }
9179
9180 static void
9181 learner_build_free(struct rte_swx_pipeline *p)
9182 {
9183         uint32_t i;
9184
9185         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9186                 struct thread *t = &p->threads[i];
9187                 uint32_t j;
9188
9189                 if (!t->learners)
9190                         continue;
9191
9192                 for (j = 0; j < p->n_learners; j++) {
9193                         struct learner_runtime *r = &t->learners[j];
9194
9195                         free(r->mailbox);
9196                         free(r->action_data);
9197                 }
9198
9199                 free(t->learners);
9200                 t->learners = NULL;
9201         }
9202
9203         if (p->learner_stats) {
9204                 for (i = 0; i < p->n_learners; i++)
9205                         free(p->learner_stats[i].n_pkts_action);
9206
9207                 free(p->learner_stats);
9208         }
9209 }
9210
9211 static int
9212 learner_build(struct rte_swx_pipeline *p)
9213 {
9214         uint32_t i;
9215         int status = 0;
9216
9217         /* Per pipeline: learner statistics. */
9218         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9219         CHECK(p->learner_stats, ENOMEM);
9220
9221         for (i = 0; i < p->n_learners; i++) {
9222                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9223                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9224         }
9225
9226         /* Per thread: learner run-time. */
9227         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9228                 struct thread *t = &p->threads[i];
9229                 struct learner *l;
9230
9231                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9232                 if (!t->learners) {
9233                         status = -ENOMEM;
9234                         goto error;
9235                 }
9236
9237                 TAILQ_FOREACH(l, &p->learners, node) {
9238                         struct learner_runtime *r = &t->learners[l->id];
9239                         uint64_t size;
9240                         uint32_t j;
9241
9242                         /* r->mailbox. */
9243                         size = rte_swx_table_learner_mailbox_size_get();
9244                         if (size) {
9245                                 r->mailbox = calloc(1, size);
9246                                 if (!r->mailbox) {
9247                                         status = -ENOMEM;
9248                                         goto error;
9249                                 }
9250                         }
9251
9252                         /* r->key. */
9253                         r->key = l->header ?
9254                                 &t->structs[l->header->struct_id] :
9255                                 &t->structs[p->metadata_struct_id];
9256
9257                         /* r->action_data. */
9258                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9259                         if (!r->action_data) {
9260                                 status = -ENOMEM;
9261                                 goto error;
9262                         }
9263
9264                         for (j = 0; j < l->n_actions; j++) {
9265                                 struct action *a = l->actions[j];
9266                                 struct field *mf = l->action_arg[j];
9267                                 uint8_t *m = t->structs[p->metadata_struct_id];
9268
9269                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9270                         }
9271                 }
9272         }
9273
9274         return 0;
9275
9276 error:
9277         learner_build_free(p);
9278         return status;
9279 }
9280
9281 static void
9282 learner_free(struct rte_swx_pipeline *p)
9283 {
9284         learner_build_free(p);
9285
9286         /* Learner tables. */
9287         for ( ; ; ) {
9288                 struct learner *l;
9289
9290                 l = TAILQ_FIRST(&p->learners);
9291                 if (!l)
9292                         break;
9293
9294                 TAILQ_REMOVE(&p->learners, l, node);
9295                 free(l->fields);
9296                 free(l->actions);
9297                 free(l->action_arg);
9298                 free(l->default_action_data);
9299                 free(l);
9300         }
9301 }
9302
9303 /*
9304  * Table state.
9305  */
9306 static int
9307 table_state_build(struct rte_swx_pipeline *p)
9308 {
9309         struct table *table;
9310         struct selector *s;
9311         struct learner *l;
9312
9313         p->table_state = calloc(p->n_tables + p->n_selectors,
9314                                 sizeof(struct rte_swx_table_state));
9315         CHECK(p->table_state, ENOMEM);
9316
9317         TAILQ_FOREACH(table, &p->tables, node) {
9318                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9319
9320                 if (table->type) {
9321                         struct rte_swx_table_params *params;
9322
9323                         /* ts->obj. */
9324                         params = table_params_get(table);
9325                         CHECK(params, ENOMEM);
9326
9327                         ts->obj = table->type->ops.create(params,
9328                                 NULL,
9329                                 table->args,
9330                                 p->numa_node);
9331
9332                         table_params_free(params);
9333                         CHECK(ts->obj, ENODEV);
9334                 }
9335
9336                 /* ts->default_action_data. */
9337                 if (table->action_data_size_max) {
9338                         ts->default_action_data =
9339                                 malloc(table->action_data_size_max);
9340                         CHECK(ts->default_action_data, ENOMEM);
9341
9342                         memcpy(ts->default_action_data,
9343                                table->default_action_data,
9344                                table->action_data_size_max);
9345                 }
9346
9347                 /* ts->default_action_id. */
9348                 ts->default_action_id = table->default_action->id;
9349         }
9350
9351         TAILQ_FOREACH(s, &p->selectors, node) {
9352                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9353                 struct rte_swx_table_selector_params *params;
9354
9355                 /* ts->obj. */
9356                 params = selector_table_params_get(s);
9357                 CHECK(params, ENOMEM);
9358
9359                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9360
9361                 selector_params_free(params);
9362                 CHECK(ts->obj, ENODEV);
9363         }
9364
9365         TAILQ_FOREACH(l, &p->learners, node) {
9366                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9367                         p->n_selectors + l->id];
9368                 struct rte_swx_table_learner_params *params;
9369
9370                 /* ts->obj. */
9371                 params = learner_params_get(l);
9372                 CHECK(params, ENOMEM);
9373
9374                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9375                 learner_params_free(params);
9376                 CHECK(ts->obj, ENODEV);
9377
9378                 /* ts->default_action_data. */
9379                 if (l->action_data_size_max) {
9380                         ts->default_action_data = malloc(l->action_data_size_max);
9381                         CHECK(ts->default_action_data, ENOMEM);
9382
9383                         memcpy(ts->default_action_data,
9384                                l->default_action_data,
9385                                l->action_data_size_max);
9386                 }
9387
9388                 /* ts->default_action_id. */
9389                 ts->default_action_id = l->default_action->id;
9390         }
9391
9392         return 0;
9393 }
9394
9395 static void
9396 table_state_build_free(struct rte_swx_pipeline *p)
9397 {
9398         uint32_t i;
9399
9400         if (!p->table_state)
9401                 return;
9402
9403         for (i = 0; i < p->n_tables; i++) {
9404                 struct rte_swx_table_state *ts = &p->table_state[i];
9405                 struct table *table = table_find_by_id(p, i);
9406
9407                 /* ts->obj. */
9408                 if (table->type && ts->obj)
9409                         table->type->ops.free(ts->obj);
9410
9411                 /* ts->default_action_data. */
9412                 free(ts->default_action_data);
9413         }
9414
9415         for (i = 0; i < p->n_selectors; i++) {
9416                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9417
9418                 /* ts->obj. */
9419                 if (ts->obj)
9420                         rte_swx_table_selector_free(ts->obj);
9421         }
9422
9423         for (i = 0; i < p->n_learners; i++) {
9424                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9425
9426                 /* ts->obj. */
9427                 if (ts->obj)
9428                         rte_swx_table_learner_free(ts->obj);
9429
9430                 /* ts->default_action_data. */
9431                 free(ts->default_action_data);
9432         }
9433
9434         free(p->table_state);
9435         p->table_state = NULL;
9436 }
9437
9438 static void
9439 table_state_free(struct rte_swx_pipeline *p)
9440 {
9441         table_state_build_free(p);
9442 }
9443
9444 /*
9445  * Register array.
9446  */
9447 static struct regarray *
9448 regarray_find(struct rte_swx_pipeline *p, const char *name)
9449 {
9450         struct regarray *elem;
9451
9452         TAILQ_FOREACH(elem, &p->regarrays, node)
9453                 if (!strcmp(elem->name, name))
9454                         return elem;
9455
9456         return NULL;
9457 }
9458
9459 static struct regarray *
9460 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9461 {
9462         struct regarray *elem = NULL;
9463
9464         TAILQ_FOREACH(elem, &p->regarrays, node)
9465                 if (elem->id == id)
9466                         return elem;
9467
9468         return NULL;
9469 }
9470
9471 int
9472 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9473                               const char *name,
9474                               uint32_t size,
9475                               uint64_t init_val)
9476 {
9477         struct regarray *r;
9478
9479         CHECK(p, EINVAL);
9480
9481         CHECK_NAME(name, EINVAL);
9482         CHECK(!regarray_find(p, name), EEXIST);
9483
9484         CHECK(size, EINVAL);
9485         size = rte_align32pow2(size);
9486
9487         /* Memory allocation. */
9488         r = calloc(1, sizeof(struct regarray));
9489         CHECK(r, ENOMEM);
9490
9491         /* Node initialization. */
9492         strcpy(r->name, name);
9493         r->init_val = init_val;
9494         r->size = size;
9495         r->id = p->n_regarrays;
9496
9497         /* Node add to tailq. */
9498         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9499         p->n_regarrays++;
9500
9501         return 0;
9502 }
9503
9504 static int
9505 regarray_build(struct rte_swx_pipeline *p)
9506 {
9507         struct regarray *regarray;
9508
9509         if (!p->n_regarrays)
9510                 return 0;
9511
9512         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9513         CHECK(p->regarray_runtime, ENOMEM);
9514
9515         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9516                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9517                 uint32_t i;
9518
9519                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9520                                          RTE_CACHE_LINE_SIZE,
9521                                          p->numa_node);
9522                 CHECK(r->regarray, ENOMEM);
9523
9524                 if (regarray->init_val)
9525                         for (i = 0; i < regarray->size; i++)
9526                                 r->regarray[i] = regarray->init_val;
9527
9528                 r->size_mask = regarray->size - 1;
9529         }
9530
9531         return 0;
9532 }
9533
9534 static void
9535 regarray_build_free(struct rte_swx_pipeline *p)
9536 {
9537         uint32_t i;
9538
9539         if (!p->regarray_runtime)
9540                 return;
9541
9542         for (i = 0; i < p->n_regarrays; i++) {
9543                 struct regarray *regarray = regarray_find_by_id(p, i);
9544                 struct regarray_runtime *r = &p->regarray_runtime[i];
9545
9546                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9547         }
9548
9549         free(p->regarray_runtime);
9550         p->regarray_runtime = NULL;
9551 }
9552
9553 static void
9554 regarray_free(struct rte_swx_pipeline *p)
9555 {
9556         regarray_build_free(p);
9557
9558         for ( ; ; ) {
9559                 struct regarray *elem;
9560
9561                 elem = TAILQ_FIRST(&p->regarrays);
9562                 if (!elem)
9563                         break;
9564
9565                 TAILQ_REMOVE(&p->regarrays, elem, node);
9566                 free(elem);
9567         }
9568 }
9569
9570 /*
9571  * Meter array.
9572  */
9573 static struct meter_profile *
9574 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9575 {
9576         struct meter_profile *elem;
9577
9578         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9579                 if (!strcmp(elem->name, name))
9580                         return elem;
9581
9582         return NULL;
9583 }
9584
9585 static struct metarray *
9586 metarray_find(struct rte_swx_pipeline *p, const char *name)
9587 {
9588         struct metarray *elem;
9589
9590         TAILQ_FOREACH(elem, &p->metarrays, node)
9591                 if (!strcmp(elem->name, name))
9592                         return elem;
9593
9594         return NULL;
9595 }
9596
9597 static struct metarray *
9598 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9599 {
9600         struct metarray *elem = NULL;
9601
9602         TAILQ_FOREACH(elem, &p->metarrays, node)
9603                 if (elem->id == id)
9604                         return elem;
9605
9606         return NULL;
9607 }
9608
9609 int
9610 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9611                                  const char *name,
9612                                  uint32_t size)
9613 {
9614         struct metarray *m;
9615
9616         CHECK(p, EINVAL);
9617
9618         CHECK_NAME(name, EINVAL);
9619         CHECK(!metarray_find(p, name), EEXIST);
9620
9621         CHECK(size, EINVAL);
9622         size = rte_align32pow2(size);
9623
9624         /* Memory allocation. */
9625         m = calloc(1, sizeof(struct metarray));
9626         CHECK(m, ENOMEM);
9627
9628         /* Node initialization. */
9629         strcpy(m->name, name);
9630         m->size = size;
9631         m->id = p->n_metarrays;
9632
9633         /* Node add to tailq. */
9634         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9635         p->n_metarrays++;
9636
9637         return 0;
9638 }
9639
9640 struct meter_profile meter_profile_default = {
9641         .node = {0},
9642         .name = "",
9643         .params = {0},
9644
9645         .profile = {
9646                 .cbs = 10000,
9647                 .pbs = 10000,
9648                 .cir_period = 1,
9649                 .cir_bytes_per_period = 1,
9650                 .pir_period = 1,
9651                 .pir_bytes_per_period = 1,
9652         },
9653
9654         .n_users = 0,
9655 };
9656
9657 static void
9658 meter_init(struct meter *m)
9659 {
9660         memset(m, 0, sizeof(struct meter));
9661         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9662         m->profile = &meter_profile_default;
9663         m->color_mask = RTE_COLOR_GREEN;
9664
9665         meter_profile_default.n_users++;
9666 }
9667
9668 static int
9669 metarray_build(struct rte_swx_pipeline *p)
9670 {
9671         struct metarray *m;
9672
9673         if (!p->n_metarrays)
9674                 return 0;
9675
9676         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9677         CHECK(p->metarray_runtime, ENOMEM);
9678
9679         TAILQ_FOREACH(m, &p->metarrays, node) {
9680                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9681                 uint32_t i;
9682
9683                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9684                                          RTE_CACHE_LINE_SIZE,
9685                                          p->numa_node);
9686                 CHECK(r->metarray, ENOMEM);
9687
9688                 for (i = 0; i < m->size; i++)
9689                         meter_init(&r->metarray[i]);
9690
9691                 r->size_mask = m->size - 1;
9692         }
9693
9694         return 0;
9695 }
9696
9697 static void
9698 metarray_build_free(struct rte_swx_pipeline *p)
9699 {
9700         uint32_t i;
9701
9702         if (!p->metarray_runtime)
9703                 return;
9704
9705         for (i = 0; i < p->n_metarrays; i++) {
9706                 struct metarray *m = metarray_find_by_id(p, i);
9707                 struct metarray_runtime *r = &p->metarray_runtime[i];
9708
9709                 env_free(r->metarray, m->size * sizeof(struct meter));
9710         }
9711
9712         free(p->metarray_runtime);
9713         p->metarray_runtime = NULL;
9714 }
9715
9716 static void
9717 metarray_free(struct rte_swx_pipeline *p)
9718 {
9719         metarray_build_free(p);
9720
9721         /* Meter arrays. */
9722         for ( ; ; ) {
9723                 struct metarray *elem;
9724
9725                 elem = TAILQ_FIRST(&p->metarrays);
9726                 if (!elem)
9727                         break;
9728
9729                 TAILQ_REMOVE(&p->metarrays, elem, node);
9730                 free(elem);
9731         }
9732
9733         /* Meter profiles. */
9734         for ( ; ; ) {
9735                 struct meter_profile *elem;
9736
9737                 elem = TAILQ_FIRST(&p->meter_profiles);
9738                 if (!elem)
9739                         break;
9740
9741                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9742                 free(elem);
9743         }
9744 }
9745
9746 /*
9747  * Pipeline.
9748  */
9749 int
9750 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9751 {
9752         struct rte_swx_pipeline *pipeline;
9753
9754         /* Check input parameters. */
9755         CHECK(p, EINVAL);
9756
9757         /* Memory allocation. */
9758         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9759         CHECK(pipeline, ENOMEM);
9760
9761         /* Initialization. */
9762         TAILQ_INIT(&pipeline->struct_types);
9763         TAILQ_INIT(&pipeline->port_in_types);
9764         TAILQ_INIT(&pipeline->ports_in);
9765         TAILQ_INIT(&pipeline->port_out_types);
9766         TAILQ_INIT(&pipeline->ports_out);
9767         TAILQ_INIT(&pipeline->extern_types);
9768         TAILQ_INIT(&pipeline->extern_objs);
9769         TAILQ_INIT(&pipeline->extern_funcs);
9770         TAILQ_INIT(&pipeline->headers);
9771         TAILQ_INIT(&pipeline->actions);
9772         TAILQ_INIT(&pipeline->table_types);
9773         TAILQ_INIT(&pipeline->tables);
9774         TAILQ_INIT(&pipeline->selectors);
9775         TAILQ_INIT(&pipeline->learners);
9776         TAILQ_INIT(&pipeline->regarrays);
9777         TAILQ_INIT(&pipeline->meter_profiles);
9778         TAILQ_INIT(&pipeline->metarrays);
9779
9780         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9781         pipeline->numa_node = numa_node;
9782
9783         *p = pipeline;
9784         return 0;
9785 }
9786
9787 void
9788 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9789 {
9790         if (!p)
9791                 return;
9792
9793         free(p->instructions);
9794
9795         metarray_free(p);
9796         regarray_free(p);
9797         table_state_free(p);
9798         learner_free(p);
9799         selector_free(p);
9800         table_free(p);
9801         action_free(p);
9802         metadata_free(p);
9803         header_free(p);
9804         extern_func_free(p);
9805         extern_obj_free(p);
9806         port_out_free(p);
9807         port_in_free(p);
9808         struct_free(p);
9809
9810         free(p);
9811 }
9812
9813 int
9814 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9815                                      const char **instructions,
9816                                      uint32_t n_instructions)
9817 {
9818         int err;
9819         uint32_t i;
9820
9821         err = instruction_config(p, NULL, instructions, n_instructions);
9822         if (err)
9823                 return err;
9824
9825         /* Thread instruction pointer reset. */
9826         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9827                 struct thread *t = &p->threads[i];
9828
9829                 thread_ip_reset(p, t);
9830         }
9831
9832         return 0;
9833 }
9834
9835 int
9836 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9837 {
9838         int status;
9839
9840         CHECK(p, EINVAL);
9841         CHECK(p->build_done == 0, EEXIST);
9842
9843         status = port_in_build(p);
9844         if (status)
9845                 goto error;
9846
9847         status = port_out_build(p);
9848         if (status)
9849                 goto error;
9850
9851         status = struct_build(p);
9852         if (status)
9853                 goto error;
9854
9855         status = extern_obj_build(p);
9856         if (status)
9857                 goto error;
9858
9859         status = extern_func_build(p);
9860         if (status)
9861                 goto error;
9862
9863         status = header_build(p);
9864         if (status)
9865                 goto error;
9866
9867         status = metadata_build(p);
9868         if (status)
9869                 goto error;
9870
9871         status = action_build(p);
9872         if (status)
9873                 goto error;
9874
9875         status = table_build(p);
9876         if (status)
9877                 goto error;
9878
9879         status = selector_build(p);
9880         if (status)
9881                 goto error;
9882
9883         status = learner_build(p);
9884         if (status)
9885                 goto error;
9886
9887         status = table_state_build(p);
9888         if (status)
9889                 goto error;
9890
9891         status = regarray_build(p);
9892         if (status)
9893                 goto error;
9894
9895         status = metarray_build(p);
9896         if (status)
9897                 goto error;
9898
9899         p->build_done = 1;
9900         return 0;
9901
9902 error:
9903         metarray_build_free(p);
9904         regarray_build_free(p);
9905         table_state_build_free(p);
9906         learner_build_free(p);
9907         selector_build_free(p);
9908         table_build_free(p);
9909         action_build_free(p);
9910         metadata_build_free(p);
9911         header_build_free(p);
9912         extern_func_build_free(p);
9913         extern_obj_build_free(p);
9914         port_out_build_free(p);
9915         port_in_build_free(p);
9916         struct_build_free(p);
9917
9918         return status;
9919 }
9920
9921 void
9922 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9923 {
9924         uint32_t i;
9925
9926         for (i = 0; i < n_instructions; i++)
9927                 instr_exec(p);
9928 }
9929
9930 void
9931 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9932 {
9933         uint32_t i;
9934
9935         for (i = 0; i < p->n_ports_out; i++) {
9936                 struct port_out_runtime *port = &p->out[i];
9937
9938                 if (port->flush)
9939                         port->flush(port->obj);
9940         }
9941 }
9942
9943 /*
9944  * Control.
9945  */
9946 int
9947 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9948                               struct rte_swx_ctl_pipeline_info *pipeline)
9949 {
9950         struct action *action;
9951         struct table *table;
9952         uint32_t n_actions = 0, n_tables = 0;
9953
9954         if (!p || !pipeline)
9955                 return -EINVAL;
9956
9957         TAILQ_FOREACH(action, &p->actions, node)
9958                 n_actions++;
9959
9960         TAILQ_FOREACH(table, &p->tables, node)
9961                 n_tables++;
9962
9963         pipeline->n_ports_in = p->n_ports_in;
9964         pipeline->n_ports_out = p->n_ports_out;
9965         pipeline->n_actions = n_actions;
9966         pipeline->n_tables = n_tables;
9967         pipeline->n_selectors = p->n_selectors;
9968         pipeline->n_learners = p->n_learners;
9969         pipeline->n_regarrays = p->n_regarrays;
9970         pipeline->n_metarrays = p->n_metarrays;
9971
9972         return 0;
9973 }
9974
9975 int
9976 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9977 {
9978         if (!p || !numa_node)
9979                 return -EINVAL;
9980
9981         *numa_node = p->numa_node;
9982         return 0;
9983 }
9984
9985 int
9986 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9987                             uint32_t action_id,
9988                             struct rte_swx_ctl_action_info *action)
9989 {
9990         struct action *a = NULL;
9991
9992         if (!p || (action_id >= p->n_actions) || !action)
9993                 return -EINVAL;
9994
9995         a = action_find_by_id(p, action_id);
9996         if (!a)
9997                 return -EINVAL;
9998
9999         strcpy(action->name, a->name);
10000         action->n_args = a->st ? a->st->n_fields : 0;
10001         return 0;
10002 }
10003
10004 int
10005 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10006                                 uint32_t action_id,
10007                                 uint32_t action_arg_id,
10008                                 struct rte_swx_ctl_action_arg_info *action_arg)
10009 {
10010         struct action *a = NULL;
10011         struct field *arg = NULL;
10012
10013         if (!p || (action_id >= p->n_actions) || !action_arg)
10014                 return -EINVAL;
10015
10016         a = action_find_by_id(p, action_id);
10017         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10018                 return -EINVAL;
10019
10020         arg = &a->st->fields[action_arg_id];
10021         strcpy(action_arg->name, arg->name);
10022         action_arg->n_bits = arg->n_bits;
10023         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10024
10025         return 0;
10026 }
10027
10028 int
10029 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10030                            uint32_t table_id,
10031                            struct rte_swx_ctl_table_info *table)
10032 {
10033         struct table *t = NULL;
10034
10035         if (!p || !table)
10036                 return -EINVAL;
10037
10038         t = table_find_by_id(p, table_id);
10039         if (!t)
10040                 return -EINVAL;
10041
10042         strcpy(table->name, t->name);
10043         strcpy(table->args, t->args);
10044         table->n_match_fields = t->n_fields;
10045         table->n_actions = t->n_actions;
10046         table->default_action_is_const = t->default_action_is_const;
10047         table->size = t->size;
10048         return 0;
10049 }
10050
10051 int
10052 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10053         uint32_t table_id,
10054         uint32_t match_field_id,
10055         struct rte_swx_ctl_table_match_field_info *match_field)
10056 {
10057         struct table *t;
10058         struct match_field *f;
10059
10060         if (!p || (table_id >= p->n_tables) || !match_field)
10061                 return -EINVAL;
10062
10063         t = table_find_by_id(p, table_id);
10064         if (!t || (match_field_id >= t->n_fields))
10065                 return -EINVAL;
10066
10067         f = &t->fields[match_field_id];
10068         match_field->match_type = f->match_type;
10069         match_field->is_header = t->header ? 1 : 0;
10070         match_field->n_bits = f->field->n_bits;
10071         match_field->offset = f->field->offset;
10072
10073         return 0;
10074 }
10075
10076 int
10077 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10078         uint32_t table_id,
10079         uint32_t table_action_id,
10080         struct rte_swx_ctl_table_action_info *table_action)
10081 {
10082         struct table *t;
10083
10084         if (!p || (table_id >= p->n_tables) || !table_action)
10085                 return -EINVAL;
10086
10087         t = table_find_by_id(p, table_id);
10088         if (!t || (table_action_id >= t->n_actions))
10089                 return -EINVAL;
10090
10091         table_action->action_id = t->actions[table_action_id]->id;
10092
10093         return 0;
10094 }
10095
10096 int
10097 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10098                           uint32_t table_id,
10099                           struct rte_swx_table_ops *table_ops,
10100                           int *is_stub)
10101 {
10102         struct table *t;
10103
10104         if (!p || (table_id >= p->n_tables))
10105                 return -EINVAL;
10106
10107         t = table_find_by_id(p, table_id);
10108         if (!t)
10109                 return -EINVAL;
10110
10111         if (t->type) {
10112                 if (table_ops)
10113                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10114                 *is_stub = 0;
10115         } else {
10116                 *is_stub = 1;
10117         }
10118
10119         return 0;
10120 }
10121
10122 int
10123 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10124                               uint32_t selector_id,
10125                               struct rte_swx_ctl_selector_info *selector)
10126 {
10127         struct selector *s = NULL;
10128
10129         if (!p || !selector)
10130                 return -EINVAL;
10131
10132         s = selector_find_by_id(p, selector_id);
10133         if (!s)
10134                 return -EINVAL;
10135
10136         strcpy(selector->name, s->name);
10137
10138         selector->n_selector_fields = s->n_selector_fields;
10139         selector->n_groups_max = s->n_groups_max;
10140         selector->n_members_per_group_max = s->n_members_per_group_max;
10141
10142         return 0;
10143 }
10144
10145 int
10146 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10147          uint32_t selector_id,
10148          struct rte_swx_ctl_table_match_field_info *field)
10149 {
10150         struct selector *s;
10151
10152         if (!p || (selector_id >= p->n_selectors) || !field)
10153                 return -EINVAL;
10154
10155         s = selector_find_by_id(p, selector_id);
10156         if (!s)
10157                 return -EINVAL;
10158
10159         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10160         field->is_header = 0;
10161         field->n_bits = s->group_id_field->n_bits;
10162         field->offset = s->group_id_field->offset;
10163
10164         return 0;
10165 }
10166
10167 int
10168 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10169          uint32_t selector_id,
10170          uint32_t selector_field_id,
10171          struct rte_swx_ctl_table_match_field_info *field)
10172 {
10173         struct selector *s;
10174         struct field *f;
10175
10176         if (!p || (selector_id >= p->n_selectors) || !field)
10177                 return -EINVAL;
10178
10179         s = selector_find_by_id(p, selector_id);
10180         if (!s || (selector_field_id >= s->n_selector_fields))
10181                 return -EINVAL;
10182
10183         f = s->selector_fields[selector_field_id];
10184         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10185         field->is_header = s->selector_header ? 1 : 0;
10186         field->n_bits = f->n_bits;
10187         field->offset = f->offset;
10188
10189         return 0;
10190 }
10191
10192 int
10193 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10194          uint32_t selector_id,
10195          struct rte_swx_ctl_table_match_field_info *field)
10196 {
10197         struct selector *s;
10198
10199         if (!p || (selector_id >= p->n_selectors) || !field)
10200                 return -EINVAL;
10201
10202         s = selector_find_by_id(p, selector_id);
10203         if (!s)
10204                 return -EINVAL;
10205
10206         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10207         field->is_header = 0;
10208         field->n_bits = s->member_id_field->n_bits;
10209         field->offset = s->member_id_field->offset;
10210
10211         return 0;
10212 }
10213
10214 int
10215 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10216                              uint32_t learner_id,
10217                              struct rte_swx_ctl_learner_info *learner)
10218 {
10219         struct learner *l = NULL;
10220
10221         if (!p || !learner)
10222                 return -EINVAL;
10223
10224         l = learner_find_by_id(p, learner_id);
10225         if (!l)
10226                 return -EINVAL;
10227
10228         strcpy(learner->name, l->name);
10229
10230         learner->n_match_fields = l->n_fields;
10231         learner->n_actions = l->n_actions;
10232         learner->default_action_is_const = l->default_action_is_const;
10233         learner->size = l->size;
10234
10235         return 0;
10236 }
10237
10238 int
10239 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10240                                          uint32_t learner_id,
10241                                          uint32_t match_field_id,
10242                                          struct rte_swx_ctl_table_match_field_info *match_field)
10243 {
10244         struct learner *l;
10245         struct field *f;
10246
10247         if (!p || (learner_id >= p->n_learners) || !match_field)
10248                 return -EINVAL;
10249
10250         l = learner_find_by_id(p, learner_id);
10251         if (!l || (match_field_id >= l->n_fields))
10252                 return -EINVAL;
10253
10254         f = l->fields[match_field_id];
10255         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10256         match_field->is_header = l->header ? 1 : 0;
10257         match_field->n_bits = f->n_bits;
10258         match_field->offset = f->offset;
10259
10260         return 0;
10261 }
10262
10263 int
10264 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10265                                     uint32_t learner_id,
10266                                     uint32_t learner_action_id,
10267                                     struct rte_swx_ctl_table_action_info *learner_action)
10268 {
10269         struct learner *l;
10270
10271         if (!p || (learner_id >= p->n_learners) || !learner_action)
10272                 return -EINVAL;
10273
10274         l = learner_find_by_id(p, learner_id);
10275         if (!l || (learner_action_id >= l->n_actions))
10276                 return -EINVAL;
10277
10278         learner_action->action_id = l->actions[learner_action_id]->id;
10279
10280         return 0;
10281 }
10282
10283 int
10284 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10285                                  struct rte_swx_table_state **table_state)
10286 {
10287         if (!p || !table_state || !p->build_done)
10288                 return -EINVAL;
10289
10290         *table_state = p->table_state;
10291         return 0;
10292 }
10293
10294 int
10295 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10296                                  struct rte_swx_table_state *table_state)
10297 {
10298         if (!p || !table_state || !p->build_done)
10299                 return -EINVAL;
10300
10301         p->table_state = table_state;
10302         return 0;
10303 }
10304
10305 int
10306 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10307                                         uint32_t port_id,
10308                                         struct rte_swx_port_in_stats *stats)
10309 {
10310         struct port_in *port;
10311
10312         if (!p || !stats)
10313                 return -EINVAL;
10314
10315         port = port_in_find(p, port_id);
10316         if (!port)
10317                 return -EINVAL;
10318
10319         port->type->ops.stats_read(port->obj, stats);
10320         return 0;
10321 }
10322
10323 int
10324 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10325                                          uint32_t port_id,
10326                                          struct rte_swx_port_out_stats *stats)
10327 {
10328         struct port_out *port;
10329
10330         if (!p || !stats)
10331                 return -EINVAL;
10332
10333         port = port_out_find(p, port_id);
10334         if (!port)
10335                 return -EINVAL;
10336
10337         port->type->ops.stats_read(port->obj, stats);
10338         return 0;
10339 }
10340
10341 int
10342 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10343                                       const char *table_name,
10344                                       struct rte_swx_table_stats *stats)
10345 {
10346         struct table *table;
10347         struct table_statistics *table_stats;
10348
10349         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10350                 return -EINVAL;
10351
10352         table = table_find(p, table_name);
10353         if (!table)
10354                 return -EINVAL;
10355
10356         table_stats = &p->table_stats[table->id];
10357
10358         memcpy(stats->n_pkts_action,
10359                table_stats->n_pkts_action,
10360                p->n_actions * sizeof(uint64_t));
10361
10362         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10363         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10364
10365         return 0;
10366 }
10367
10368 int
10369 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10370         const char *selector_name,
10371         struct rte_swx_pipeline_selector_stats *stats)
10372 {
10373         struct selector *s;
10374
10375         if (!p || !selector_name || !selector_name[0] || !stats)
10376                 return -EINVAL;
10377
10378         s = selector_find(p, selector_name);
10379         if (!s)
10380                 return -EINVAL;
10381
10382         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10383
10384         return 0;
10385 }
10386
10387 int
10388 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10389                                         const char *learner_name,
10390                                         struct rte_swx_learner_stats *stats)
10391 {
10392         struct learner *l;
10393         struct learner_statistics *learner_stats;
10394
10395         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10396                 return -EINVAL;
10397
10398         l = learner_find(p, learner_name);
10399         if (!l)
10400                 return -EINVAL;
10401
10402         learner_stats = &p->learner_stats[l->id];
10403
10404         memcpy(stats->n_pkts_action,
10405                learner_stats->n_pkts_action,
10406                p->n_actions * sizeof(uint64_t));
10407
10408         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10409         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10410
10411         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10412         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10413
10414         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10415
10416         return 0;
10417 }
10418
10419 int
10420 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10421                               uint32_t regarray_id,
10422                               struct rte_swx_ctl_regarray_info *regarray)
10423 {
10424         struct regarray *r;
10425
10426         if (!p || !regarray)
10427                 return -EINVAL;
10428
10429         r = regarray_find_by_id(p, regarray_id);
10430         if (!r)
10431                 return -EINVAL;
10432
10433         strcpy(regarray->name, r->name);
10434         regarray->size = r->size;
10435         return 0;
10436 }
10437
10438 int
10439 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10440                                    const char *regarray_name,
10441                                    uint32_t regarray_index,
10442                                    uint64_t *value)
10443 {
10444         struct regarray *regarray;
10445         struct regarray_runtime *r;
10446
10447         if (!p || !regarray_name || !value)
10448                 return -EINVAL;
10449
10450         regarray = regarray_find(p, regarray_name);
10451         if (!regarray || (regarray_index >= regarray->size))
10452                 return -EINVAL;
10453
10454         r = &p->regarray_runtime[regarray->id];
10455         *value = r->regarray[regarray_index];
10456         return 0;
10457 }
10458
10459 int
10460 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10461                                    const char *regarray_name,
10462                                    uint32_t regarray_index,
10463                                    uint64_t value)
10464 {
10465         struct regarray *regarray;
10466         struct regarray_runtime *r;
10467
10468         if (!p || !regarray_name)
10469                 return -EINVAL;
10470
10471         regarray = regarray_find(p, regarray_name);
10472         if (!regarray || (regarray_index >= regarray->size))
10473                 return -EINVAL;
10474
10475         r = &p->regarray_runtime[regarray->id];
10476         r->regarray[regarray_index] = value;
10477         return 0;
10478 }
10479
10480 int
10481 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10482                               uint32_t metarray_id,
10483                               struct rte_swx_ctl_metarray_info *metarray)
10484 {
10485         struct metarray *m;
10486
10487         if (!p || !metarray)
10488                 return -EINVAL;
10489
10490         m = metarray_find_by_id(p, metarray_id);
10491         if (!m)
10492                 return -EINVAL;
10493
10494         strcpy(metarray->name, m->name);
10495         metarray->size = m->size;
10496         return 0;
10497 }
10498
10499 int
10500 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10501                               const char *name,
10502                               struct rte_meter_trtcm_params *params)
10503 {
10504         struct meter_profile *mp;
10505         int status;
10506
10507         CHECK(p, EINVAL);
10508         CHECK_NAME(name, EINVAL);
10509         CHECK(params, EINVAL);
10510         CHECK(!meter_profile_find(p, name), EEXIST);
10511
10512         /* Node allocation. */
10513         mp = calloc(1, sizeof(struct meter_profile));
10514         CHECK(mp, ENOMEM);
10515
10516         /* Node initialization. */
10517         strcpy(mp->name, name);
10518         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10519         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10520         if (status) {
10521                 free(mp);
10522                 CHECK(0, EINVAL);
10523         }
10524
10525         /* Node add to tailq. */
10526         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10527
10528         return 0;
10529 }
10530
10531 int
10532 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10533                                  const char *name)
10534 {
10535         struct meter_profile *mp;
10536
10537         CHECK(p, EINVAL);
10538         CHECK_NAME(name, EINVAL);
10539
10540         mp = meter_profile_find(p, name);
10541         CHECK(mp, EINVAL);
10542         CHECK(!mp->n_users, EBUSY);
10543
10544         /* Remove node from tailq. */
10545         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10546         free(mp);
10547
10548         return 0;
10549 }
10550
10551 int
10552 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10553                         const char *metarray_name,
10554                         uint32_t metarray_index)
10555 {
10556         struct meter_profile *mp_old;
10557         struct metarray *metarray;
10558         struct metarray_runtime *metarray_runtime;
10559         struct meter *m;
10560
10561         CHECK(p, EINVAL);
10562         CHECK_NAME(metarray_name, EINVAL);
10563
10564         metarray = metarray_find(p, metarray_name);
10565         CHECK(metarray, EINVAL);
10566         CHECK(metarray_index < metarray->size, EINVAL);
10567
10568         metarray_runtime = &p->metarray_runtime[metarray->id];
10569         m = &metarray_runtime->metarray[metarray_index];
10570         mp_old = m->profile;
10571
10572         meter_init(m);
10573
10574         mp_old->n_users--;
10575
10576         return 0;
10577 }
10578
10579 int
10580 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10581                       const char *metarray_name,
10582                       uint32_t metarray_index,
10583                       const char *profile_name)
10584 {
10585         struct meter_profile *mp, *mp_old;
10586         struct metarray *metarray;
10587         struct metarray_runtime *metarray_runtime;
10588         struct meter *m;
10589
10590         CHECK(p, EINVAL);
10591         CHECK_NAME(metarray_name, EINVAL);
10592
10593         metarray = metarray_find(p, metarray_name);
10594         CHECK(metarray, EINVAL);
10595         CHECK(metarray_index < metarray->size, EINVAL);
10596
10597         mp = meter_profile_find(p, profile_name);
10598         CHECK(mp, EINVAL);
10599
10600         metarray_runtime = &p->metarray_runtime[metarray->id];
10601         m = &metarray_runtime->metarray[metarray_index];
10602         mp_old = m->profile;
10603
10604         memset(m, 0, sizeof(struct meter));
10605         rte_meter_trtcm_config(&m->m, &mp->profile);
10606         m->profile = mp;
10607         m->color_mask = RTE_COLORS;
10608
10609         mp->n_users++;
10610         mp_old->n_users--;
10611
10612         return 0;
10613 }
10614
10615 int
10616 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10617                              const char *metarray_name,
10618                              uint32_t metarray_index,
10619                              struct rte_swx_ctl_meter_stats *stats)
10620 {
10621         struct metarray *metarray;
10622         struct metarray_runtime *metarray_runtime;
10623         struct meter *m;
10624
10625         CHECK(p, EINVAL);
10626         CHECK_NAME(metarray_name, EINVAL);
10627
10628         metarray = metarray_find(p, metarray_name);
10629         CHECK(metarray, EINVAL);
10630         CHECK(metarray_index < metarray->size, EINVAL);
10631
10632         CHECK(stats, EINVAL);
10633
10634         metarray_runtime = &p->metarray_runtime[metarray->id];
10635         m = &metarray_runtime->metarray[metarray_index];
10636
10637         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10638         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10639
10640         return 0;
10641 }