lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t selector_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107         struct selector_runtime *selector = &t->selectors[selector_id];
2108         struct selector_statistics *stats = &p->selector_stats[selector_id];
2109         uint64_t n_pkts = stats->n_pkts;
2110         int done;
2111
2112         /* Table. */
2113         done = rte_swx_table_selector_select(ts->obj,
2114                            selector->mailbox,
2115                            selector->group_id_buffer,
2116                            selector->selector_buffer,
2117                            selector->member_id_buffer);
2118         if (!done) {
2119                 /* Thread. */
2120                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2121                       p->thread_id,
2122                       selector_id);
2123
2124                 thread_yield(p);
2125                 return;
2126         }
2127
2128
2129         TRACE("[Thread %2u] selector %u\n",
2130               p->thread_id,
2131               selector_id);
2132
2133         stats->n_pkts = n_pkts + 1;
2134
2135         /* Thread. */
2136         thread_ip_inc(p);
2137 }
2138
2139 static inline void
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t learner_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146                 p->n_selectors + learner_id];
2147         struct learner_runtime *l = &t->learners[learner_id];
2148         struct learner_statistics *stats = &p->learner_stats[learner_id];
2149         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150         uint8_t *action_data;
2151         int done, hit;
2152
2153         /* Table. */
2154         time = rte_get_tsc_cycles();
2155
2156         done = rte_swx_table_learner_lookup(ts->obj,
2157                                             l->mailbox,
2158                                             time,
2159                                             l->key,
2160                                             &action_id,
2161                                             &action_data,
2162                                             &hit);
2163         if (!done) {
2164                 /* Thread. */
2165                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2166                       p->thread_id,
2167                       learner_id);
2168
2169                 thread_yield(p);
2170                 return;
2171         }
2172
2173         action_id = hit ? action_id : ts->default_action_id;
2174         action_data = hit ? action_data : ts->default_action_data;
2175         n_pkts_hit = stats->n_pkts_hit[hit];
2176         n_pkts_action = stats->n_pkts_action[action_id];
2177
2178         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2179               p->thread_id,
2180               learner_id,
2181               hit ? "hit" : "miss",
2182               (uint32_t)action_id);
2183
2184         t->action_id = action_id;
2185         t->structs[0] = action_data;
2186         t->hit = hit;
2187         t->learner_id = learner_id;
2188         t->time = time;
2189         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2191
2192         /* Thread. */
2193         thread_ip_action_call(p, t, action_id);
2194 }
2195
2196 /*
2197  * learn.
2198  */
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2201
2202 static int
2203 action_has_nbo_args(struct action *a);
2204
2205 static int
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207                       struct action *action,
2208                       char **tokens,
2209                       int n_tokens,
2210                       struct instruction *instr,
2211                       struct instruction_data *data __rte_unused)
2212 {
2213         struct action *a;
2214
2215         CHECK(action, EINVAL);
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         a = action_find(p, tokens[1]);
2219         CHECK(a, EINVAL);
2220         CHECK(!action_has_nbo_args(a), EINVAL);
2221
2222         instr->type = INSTR_LEARNER_LEARN;
2223         instr->learn.action_id = a->id;
2224
2225         return 0;
2226 }
2227
2228 static inline void
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_learn_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_inc(p);
2238 }
2239
2240 /*
2241  * forget.
2242  */
2243 static int
2244 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2245                        struct action *action,
2246                        char **tokens __rte_unused,
2247                        int n_tokens,
2248                        struct instruction *instr,
2249                        struct instruction_data *data __rte_unused)
2250 {
2251         CHECK(action, EINVAL);
2252         CHECK(n_tokens == 1, EINVAL);
2253
2254         instr->type = INSTR_LEARNER_FORGET;
2255
2256         return 0;
2257 }
2258
2259 static inline void
2260 instr_forget_exec(struct rte_swx_pipeline *p)
2261 {
2262         struct thread *t = &p->threads[p->thread_id];
2263         struct instruction *ip = t->ip;
2264
2265         __instr_forget_exec(p, t, ip);
2266
2267         /* Thread. */
2268         thread_ip_inc(p);
2269 }
2270
2271 /*
2272  * extern.
2273  */
2274 static int
2275 instr_extern_translate(struct rte_swx_pipeline *p,
2276                        struct action *action __rte_unused,
2277                        char **tokens,
2278                        int n_tokens,
2279                        struct instruction *instr,
2280                        struct instruction_data *data __rte_unused)
2281 {
2282         char *token = tokens[1];
2283
2284         CHECK(n_tokens == 2, EINVAL);
2285
2286         if (token[0] == 'e') {
2287                 struct extern_obj *obj;
2288                 struct extern_type_member_func *func;
2289
2290                 func = extern_obj_member_func_parse(p, token, &obj);
2291                 CHECK(func, EINVAL);
2292
2293                 instr->type = INSTR_EXTERN_OBJ;
2294                 instr->ext_obj.ext_obj_id = obj->id;
2295                 instr->ext_obj.func_id = func->id;
2296
2297                 return 0;
2298         }
2299
2300         if (token[0] == 'f') {
2301                 struct extern_func *func;
2302
2303                 func = extern_func_parse(p, token);
2304                 CHECK(func, EINVAL);
2305
2306                 instr->type = INSTR_EXTERN_FUNC;
2307                 instr->ext_func.ext_func_id = func->id;
2308
2309                 return 0;
2310         }
2311
2312         CHECK(0, EINVAL);
2313 }
2314
2315 static inline void
2316 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2317 {
2318         struct thread *t = &p->threads[p->thread_id];
2319         struct instruction *ip = t->ip;
2320         uint32_t done;
2321
2322         /* Extern object member function execute. */
2323         done = __instr_extern_obj_exec(p, t, ip);
2324
2325         /* Thread. */
2326         thread_ip_inc_cond(t, done);
2327         thread_yield_cond(p, done ^ 1);
2328 }
2329
2330 static inline void
2331 instr_extern_func_exec(struct rte_swx_pipeline *p)
2332 {
2333         struct thread *t = &p->threads[p->thread_id];
2334         struct instruction *ip = t->ip;
2335         uint32_t done;
2336
2337         /* Extern function execute. */
2338         done = __instr_extern_func_exec(p, t, ip);
2339
2340         /* Thread. */
2341         thread_ip_inc_cond(t, done);
2342         thread_yield_cond(p, done ^ 1);
2343 }
2344
2345 /*
2346  * mov.
2347  */
2348 static int
2349 instr_mov_translate(struct rte_swx_pipeline *p,
2350                     struct action *action,
2351                     char **tokens,
2352                     int n_tokens,
2353                     struct instruction *instr,
2354                     struct instruction_data *data __rte_unused)
2355 {
2356         char *dst = tokens[1], *src = tokens[2];
2357         struct field *fdst, *fsrc;
2358         uint64_t src_val;
2359         uint32_t dst_struct_id = 0, src_struct_id = 0;
2360
2361         CHECK(n_tokens == 3, EINVAL);
2362
2363         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2364         CHECK(fdst, EINVAL);
2365         CHECK(!fdst->var_size, EINVAL);
2366
2367         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2368         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2369         if (fsrc) {
2370                 CHECK(!fsrc->var_size, EINVAL);
2371
2372                 instr->type = INSTR_MOV;
2373                 if (dst[0] != 'h' && src[0] == 'h')
2374                         instr->type = INSTR_MOV_MH;
2375                 if (dst[0] == 'h' && src[0] != 'h')
2376                         instr->type = INSTR_MOV_HM;
2377                 if (dst[0] == 'h' && src[0] == 'h')
2378                         instr->type = INSTR_MOV_HH;
2379
2380                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2381                 instr->mov.dst.n_bits = fdst->n_bits;
2382                 instr->mov.dst.offset = fdst->offset / 8;
2383                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2384                 instr->mov.src.n_bits = fsrc->n_bits;
2385                 instr->mov.src.offset = fsrc->offset / 8;
2386                 return 0;
2387         }
2388
2389         /* MOV_I. */
2390         src_val = strtoull(src, &src, 0);
2391         CHECK(!src[0], EINVAL);
2392
2393         if (dst[0] == 'h')
2394                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2395
2396         instr->type = INSTR_MOV_I;
2397         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2398         instr->mov.dst.n_bits = fdst->n_bits;
2399         instr->mov.dst.offset = fdst->offset / 8;
2400         instr->mov.src_val = src_val;
2401         return 0;
2402 }
2403
2404 static inline void
2405 instr_mov_exec(struct rte_swx_pipeline *p)
2406 {
2407         struct thread *t = &p->threads[p->thread_id];
2408         struct instruction *ip = t->ip;
2409
2410         TRACE("[Thread %2u] mov\n",
2411               p->thread_id);
2412
2413         MOV(t, ip);
2414
2415         /* Thread. */
2416         thread_ip_inc(p);
2417 }
2418
2419 static inline void
2420 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2421 {
2422         struct thread *t = &p->threads[p->thread_id];
2423         struct instruction *ip = t->ip;
2424
2425         TRACE("[Thread %2u] mov (mh)\n",
2426               p->thread_id);
2427
2428         MOV_MH(t, ip);
2429
2430         /* Thread. */
2431         thread_ip_inc(p);
2432 }
2433
2434 static inline void
2435 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2436 {
2437         struct thread *t = &p->threads[p->thread_id];
2438         struct instruction *ip = t->ip;
2439
2440         TRACE("[Thread %2u] mov (hm)\n",
2441               p->thread_id);
2442
2443         MOV_HM(t, ip);
2444
2445         /* Thread. */
2446         thread_ip_inc(p);
2447 }
2448
2449 static inline void
2450 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2451 {
2452         struct thread *t = &p->threads[p->thread_id];
2453         struct instruction *ip = t->ip;
2454
2455         TRACE("[Thread %2u] mov (hh)\n",
2456               p->thread_id);
2457
2458         MOV_HH(t, ip);
2459
2460         /* Thread. */
2461         thread_ip_inc(p);
2462 }
2463
2464 static inline void
2465 instr_mov_i_exec(struct rte_swx_pipeline *p)
2466 {
2467         struct thread *t = &p->threads[p->thread_id];
2468         struct instruction *ip = t->ip;
2469
2470         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2471               p->thread_id,
2472               ip->mov.src_val);
2473
2474         MOV_I(t, ip);
2475
2476         /* Thread. */
2477         thread_ip_inc(p);
2478 }
2479
2480 /*
2481  * dma.
2482  */
2483 static inline void
2484 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2485
2486 static inline void
2487 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2488 {
2489         struct thread *t = &p->threads[p->thread_id];
2490         struct instruction *ip = t->ip;
2491         uint8_t *action_data = t->structs[0];
2492         uint64_t valid_headers = t->valid_headers;
2493         uint32_t i;
2494
2495         for (i = 0; i < n_dma; i++) {
2496                 uint32_t header_id = ip->dma.dst.header_id[i];
2497                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2498                 uint32_t offset = ip->dma.src.offset[i];
2499                 uint32_t n_bytes = ip->dma.n_bytes[i];
2500
2501                 struct header_runtime *h = &t->headers[header_id];
2502                 uint8_t *h_ptr0 = h->ptr0;
2503                 uint8_t *h_ptr = t->structs[struct_id];
2504
2505                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2506                         h_ptr : h_ptr0;
2507                 void *src = &action_data[offset];
2508
2509                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2510
2511                 /* Headers. */
2512                 memcpy(dst, src, n_bytes);
2513                 t->structs[struct_id] = dst;
2514                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2515         }
2516
2517         t->valid_headers = valid_headers;
2518 }
2519
2520 static inline void
2521 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2522 {
2523         __instr_dma_ht_exec(p, 1);
2524
2525         /* Thread. */
2526         thread_ip_inc(p);
2527 }
2528
2529 static inline void
2530 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2531 {
2532         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2533               p->thread_id);
2534
2535         __instr_dma_ht_exec(p, 2);
2536
2537         /* Thread. */
2538         thread_ip_inc(p);
2539 }
2540
2541 static inline void
2542 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2543 {
2544         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2545               p->thread_id);
2546
2547         __instr_dma_ht_exec(p, 3);
2548
2549         /* Thread. */
2550         thread_ip_inc(p);
2551 }
2552
2553 static inline void
2554 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2555 {
2556         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2557               p->thread_id);
2558
2559         __instr_dma_ht_exec(p, 4);
2560
2561         /* Thread. */
2562         thread_ip_inc(p);
2563 }
2564
2565 static inline void
2566 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2567 {
2568         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2569               p->thread_id);
2570
2571         __instr_dma_ht_exec(p, 5);
2572
2573         /* Thread. */
2574         thread_ip_inc(p);
2575 }
2576
2577 static inline void
2578 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2579 {
2580         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2581               p->thread_id);
2582
2583         __instr_dma_ht_exec(p, 6);
2584
2585         /* Thread. */
2586         thread_ip_inc(p);
2587 }
2588
2589 static inline void
2590 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2591 {
2592         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2593               p->thread_id);
2594
2595         __instr_dma_ht_exec(p, 7);
2596
2597         /* Thread. */
2598         thread_ip_inc(p);
2599 }
2600
2601 static inline void
2602 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2603 {
2604         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2605               p->thread_id);
2606
2607         __instr_dma_ht_exec(p, 8);
2608
2609         /* Thread. */
2610         thread_ip_inc(p);
2611 }
2612
2613 /*
2614  * alu.
2615  */
2616 static int
2617 instr_alu_add_translate(struct rte_swx_pipeline *p,
2618                         struct action *action,
2619                         char **tokens,
2620                         int n_tokens,
2621                         struct instruction *instr,
2622                         struct instruction_data *data __rte_unused)
2623 {
2624         char *dst = tokens[1], *src = tokens[2];
2625         struct field *fdst, *fsrc;
2626         uint64_t src_val;
2627         uint32_t dst_struct_id = 0, src_struct_id = 0;
2628
2629         CHECK(n_tokens == 3, EINVAL);
2630
2631         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2632         CHECK(fdst, EINVAL);
2633         CHECK(!fdst->var_size, EINVAL);
2634
2635         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2636         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2637         if (fsrc) {
2638                 CHECK(!fsrc->var_size, EINVAL);
2639
2640                 instr->type = INSTR_ALU_ADD;
2641                 if (dst[0] == 'h' && src[0] != 'h')
2642                         instr->type = INSTR_ALU_ADD_HM;
2643                 if (dst[0] != 'h' && src[0] == 'h')
2644                         instr->type = INSTR_ALU_ADD_MH;
2645                 if (dst[0] == 'h' && src[0] == 'h')
2646                         instr->type = INSTR_ALU_ADD_HH;
2647
2648                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2649                 instr->alu.dst.n_bits = fdst->n_bits;
2650                 instr->alu.dst.offset = fdst->offset / 8;
2651                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2652                 instr->alu.src.n_bits = fsrc->n_bits;
2653                 instr->alu.src.offset = fsrc->offset / 8;
2654                 return 0;
2655         }
2656
2657         /* ADD_MI, ADD_HI. */
2658         src_val = strtoull(src, &src, 0);
2659         CHECK(!src[0], EINVAL);
2660
2661         instr->type = INSTR_ALU_ADD_MI;
2662         if (dst[0] == 'h')
2663                 instr->type = INSTR_ALU_ADD_HI;
2664
2665         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2666         instr->alu.dst.n_bits = fdst->n_bits;
2667         instr->alu.dst.offset = fdst->offset / 8;
2668         instr->alu.src_val = src_val;
2669         return 0;
2670 }
2671
2672 static int
2673 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2674                         struct action *action,
2675                         char **tokens,
2676                         int n_tokens,
2677                         struct instruction *instr,
2678                         struct instruction_data *data __rte_unused)
2679 {
2680         char *dst = tokens[1], *src = tokens[2];
2681         struct field *fdst, *fsrc;
2682         uint64_t src_val;
2683         uint32_t dst_struct_id = 0, src_struct_id = 0;
2684
2685         CHECK(n_tokens == 3, EINVAL);
2686
2687         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2688         CHECK(fdst, EINVAL);
2689         CHECK(!fdst->var_size, EINVAL);
2690
2691         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2692         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2693         if (fsrc) {
2694                 CHECK(!fsrc->var_size, EINVAL);
2695
2696                 instr->type = INSTR_ALU_SUB;
2697                 if (dst[0] == 'h' && src[0] != 'h')
2698                         instr->type = INSTR_ALU_SUB_HM;
2699                 if (dst[0] != 'h' && src[0] == 'h')
2700                         instr->type = INSTR_ALU_SUB_MH;
2701                 if (dst[0] == 'h' && src[0] == 'h')
2702                         instr->type = INSTR_ALU_SUB_HH;
2703
2704                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2705                 instr->alu.dst.n_bits = fdst->n_bits;
2706                 instr->alu.dst.offset = fdst->offset / 8;
2707                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2708                 instr->alu.src.n_bits = fsrc->n_bits;
2709                 instr->alu.src.offset = fsrc->offset / 8;
2710                 return 0;
2711         }
2712
2713         /* SUB_MI, SUB_HI. */
2714         src_val = strtoull(src, &src, 0);
2715         CHECK(!src[0], EINVAL);
2716
2717         instr->type = INSTR_ALU_SUB_MI;
2718         if (dst[0] == 'h')
2719                 instr->type = INSTR_ALU_SUB_HI;
2720
2721         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2722         instr->alu.dst.n_bits = fdst->n_bits;
2723         instr->alu.dst.offset = fdst->offset / 8;
2724         instr->alu.src_val = src_val;
2725         return 0;
2726 }
2727
2728 static int
2729 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2730                           struct action *action __rte_unused,
2731                           char **tokens,
2732                           int n_tokens,
2733                           struct instruction *instr,
2734                           struct instruction_data *data __rte_unused)
2735 {
2736         char *dst = tokens[1], *src = tokens[2];
2737         struct header *hdst, *hsrc;
2738         struct field *fdst, *fsrc;
2739
2740         CHECK(n_tokens == 3, EINVAL);
2741
2742         fdst = header_field_parse(p, dst, &hdst);
2743         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2744         CHECK(!fdst->var_size, EINVAL);
2745
2746         /* CKADD_FIELD. */
2747         fsrc = header_field_parse(p, src, &hsrc);
2748         if (fsrc) {
2749                 CHECK(!fsrc->var_size, EINVAL);
2750
2751                 instr->type = INSTR_ALU_CKADD_FIELD;
2752                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2753                 instr->alu.dst.n_bits = fdst->n_bits;
2754                 instr->alu.dst.offset = fdst->offset / 8;
2755                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2756                 instr->alu.src.n_bits = fsrc->n_bits;
2757                 instr->alu.src.offset = fsrc->offset / 8;
2758                 return 0;
2759         }
2760
2761         /* CKADD_STRUCT, CKADD_STRUCT20. */
2762         hsrc = header_parse(p, src);
2763         CHECK(hsrc, EINVAL);
2764         CHECK(!hsrc->st->var_size, EINVAL);
2765
2766         instr->type = INSTR_ALU_CKADD_STRUCT;
2767         if ((hsrc->st->n_bits / 8) == 20)
2768                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2769
2770         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2771         instr->alu.dst.n_bits = fdst->n_bits;
2772         instr->alu.dst.offset = fdst->offset / 8;
2773         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2774         instr->alu.src.n_bits = hsrc->st->n_bits;
2775         instr->alu.src.offset = 0; /* Unused. */
2776         return 0;
2777 }
2778
2779 static int
2780 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2781                           struct action *action __rte_unused,
2782                           char **tokens,
2783                           int n_tokens,
2784                           struct instruction *instr,
2785                           struct instruction_data *data __rte_unused)
2786 {
2787         char *dst = tokens[1], *src = tokens[2];
2788         struct header *hdst, *hsrc;
2789         struct field *fdst, *fsrc;
2790
2791         CHECK(n_tokens == 3, EINVAL);
2792
2793         fdst = header_field_parse(p, dst, &hdst);
2794         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2795         CHECK(!fdst->var_size, EINVAL);
2796
2797         fsrc = header_field_parse(p, src, &hsrc);
2798         CHECK(fsrc, EINVAL);
2799         CHECK(!fsrc->var_size, EINVAL);
2800
2801         instr->type = INSTR_ALU_CKSUB_FIELD;
2802         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2803         instr->alu.dst.n_bits = fdst->n_bits;
2804         instr->alu.dst.offset = fdst->offset / 8;
2805         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2806         instr->alu.src.n_bits = fsrc->n_bits;
2807         instr->alu.src.offset = fsrc->offset / 8;
2808         return 0;
2809 }
2810
2811 static int
2812 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2813                         struct action *action,
2814                         char **tokens,
2815                         int n_tokens,
2816                         struct instruction *instr,
2817                         struct instruction_data *data __rte_unused)
2818 {
2819         char *dst = tokens[1], *src = tokens[2];
2820         struct field *fdst, *fsrc;
2821         uint64_t src_val;
2822         uint32_t dst_struct_id = 0, src_struct_id = 0;
2823
2824         CHECK(n_tokens == 3, EINVAL);
2825
2826         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2827         CHECK(fdst, EINVAL);
2828         CHECK(!fdst->var_size, EINVAL);
2829
2830         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2831         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2832         if (fsrc) {
2833                 CHECK(!fsrc->var_size, EINVAL);
2834
2835                 instr->type = INSTR_ALU_SHL;
2836                 if (dst[0] == 'h' && src[0] != 'h')
2837                         instr->type = INSTR_ALU_SHL_HM;
2838                 if (dst[0] != 'h' && src[0] == 'h')
2839                         instr->type = INSTR_ALU_SHL_MH;
2840                 if (dst[0] == 'h' && src[0] == 'h')
2841                         instr->type = INSTR_ALU_SHL_HH;
2842
2843                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2844                 instr->alu.dst.n_bits = fdst->n_bits;
2845                 instr->alu.dst.offset = fdst->offset / 8;
2846                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2847                 instr->alu.src.n_bits = fsrc->n_bits;
2848                 instr->alu.src.offset = fsrc->offset / 8;
2849                 return 0;
2850         }
2851
2852         /* SHL_MI, SHL_HI. */
2853         src_val = strtoull(src, &src, 0);
2854         CHECK(!src[0], EINVAL);
2855
2856         instr->type = INSTR_ALU_SHL_MI;
2857         if (dst[0] == 'h')
2858                 instr->type = INSTR_ALU_SHL_HI;
2859
2860         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2861         instr->alu.dst.n_bits = fdst->n_bits;
2862         instr->alu.dst.offset = fdst->offset / 8;
2863         instr->alu.src_val = src_val;
2864         return 0;
2865 }
2866
2867 static int
2868 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2869                         struct action *action,
2870                         char **tokens,
2871                         int n_tokens,
2872                         struct instruction *instr,
2873                         struct instruction_data *data __rte_unused)
2874 {
2875         char *dst = tokens[1], *src = tokens[2];
2876         struct field *fdst, *fsrc;
2877         uint64_t src_val;
2878         uint32_t dst_struct_id = 0, src_struct_id = 0;
2879
2880         CHECK(n_tokens == 3, EINVAL);
2881
2882         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2883         CHECK(fdst, EINVAL);
2884         CHECK(!fdst->var_size, EINVAL);
2885
2886         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2887         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2888         if (fsrc) {
2889                 CHECK(!fsrc->var_size, EINVAL);
2890
2891                 instr->type = INSTR_ALU_SHR;
2892                 if (dst[0] == 'h' && src[0] != 'h')
2893                         instr->type = INSTR_ALU_SHR_HM;
2894                 if (dst[0] != 'h' && src[0] == 'h')
2895                         instr->type = INSTR_ALU_SHR_MH;
2896                 if (dst[0] == 'h' && src[0] == 'h')
2897                         instr->type = INSTR_ALU_SHR_HH;
2898
2899                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2900                 instr->alu.dst.n_bits = fdst->n_bits;
2901                 instr->alu.dst.offset = fdst->offset / 8;
2902                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2903                 instr->alu.src.n_bits = fsrc->n_bits;
2904                 instr->alu.src.offset = fsrc->offset / 8;
2905                 return 0;
2906         }
2907
2908         /* SHR_MI, SHR_HI. */
2909         src_val = strtoull(src, &src, 0);
2910         CHECK(!src[0], EINVAL);
2911
2912         instr->type = INSTR_ALU_SHR_MI;
2913         if (dst[0] == 'h')
2914                 instr->type = INSTR_ALU_SHR_HI;
2915
2916         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2917         instr->alu.dst.n_bits = fdst->n_bits;
2918         instr->alu.dst.offset = fdst->offset / 8;
2919         instr->alu.src_val = src_val;
2920         return 0;
2921 }
2922
2923 static int
2924 instr_alu_and_translate(struct rte_swx_pipeline *p,
2925                         struct action *action,
2926                         char **tokens,
2927                         int n_tokens,
2928                         struct instruction *instr,
2929                         struct instruction_data *data __rte_unused)
2930 {
2931         char *dst = tokens[1], *src = tokens[2];
2932         struct field *fdst, *fsrc;
2933         uint64_t src_val;
2934         uint32_t dst_struct_id = 0, src_struct_id = 0;
2935
2936         CHECK(n_tokens == 3, EINVAL);
2937
2938         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2939         CHECK(fdst, EINVAL);
2940         CHECK(!fdst->var_size, EINVAL);
2941
2942         /* AND, AND_MH, AND_HM, AND_HH. */
2943         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2944         if (fsrc) {
2945                 CHECK(!fsrc->var_size, EINVAL);
2946
2947                 instr->type = INSTR_ALU_AND;
2948                 if (dst[0] != 'h' && src[0] == 'h')
2949                         instr->type = INSTR_ALU_AND_MH;
2950                 if (dst[0] == 'h' && src[0] != 'h')
2951                         instr->type = INSTR_ALU_AND_HM;
2952                 if (dst[0] == 'h' && src[0] == 'h')
2953                         instr->type = INSTR_ALU_AND_HH;
2954
2955                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2956                 instr->alu.dst.n_bits = fdst->n_bits;
2957                 instr->alu.dst.offset = fdst->offset / 8;
2958                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2959                 instr->alu.src.n_bits = fsrc->n_bits;
2960                 instr->alu.src.offset = fsrc->offset / 8;
2961                 return 0;
2962         }
2963
2964         /* AND_I. */
2965         src_val = strtoull(src, &src, 0);
2966         CHECK(!src[0], EINVAL);
2967
2968         if (dst[0] == 'h')
2969                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2970
2971         instr->type = INSTR_ALU_AND_I;
2972         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2973         instr->alu.dst.n_bits = fdst->n_bits;
2974         instr->alu.dst.offset = fdst->offset / 8;
2975         instr->alu.src_val = src_val;
2976         return 0;
2977 }
2978
2979 static int
2980 instr_alu_or_translate(struct rte_swx_pipeline *p,
2981                        struct action *action,
2982                        char **tokens,
2983                        int n_tokens,
2984                        struct instruction *instr,
2985                        struct instruction_data *data __rte_unused)
2986 {
2987         char *dst = tokens[1], *src = tokens[2];
2988         struct field *fdst, *fsrc;
2989         uint64_t src_val;
2990         uint32_t dst_struct_id = 0, src_struct_id = 0;
2991
2992         CHECK(n_tokens == 3, EINVAL);
2993
2994         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2995         CHECK(fdst, EINVAL);
2996         CHECK(!fdst->var_size, EINVAL);
2997
2998         /* OR, OR_MH, OR_HM, OR_HH. */
2999         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3000         if (fsrc) {
3001                 CHECK(!fsrc->var_size, EINVAL);
3002
3003                 instr->type = INSTR_ALU_OR;
3004                 if (dst[0] != 'h' && src[0] == 'h')
3005                         instr->type = INSTR_ALU_OR_MH;
3006                 if (dst[0] == 'h' && src[0] != 'h')
3007                         instr->type = INSTR_ALU_OR_HM;
3008                 if (dst[0] == 'h' && src[0] == 'h')
3009                         instr->type = INSTR_ALU_OR_HH;
3010
3011                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3012                 instr->alu.dst.n_bits = fdst->n_bits;
3013                 instr->alu.dst.offset = fdst->offset / 8;
3014                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3015                 instr->alu.src.n_bits = fsrc->n_bits;
3016                 instr->alu.src.offset = fsrc->offset / 8;
3017                 return 0;
3018         }
3019
3020         /* OR_I. */
3021         src_val = strtoull(src, &src, 0);
3022         CHECK(!src[0], EINVAL);
3023
3024         if (dst[0] == 'h')
3025                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3026
3027         instr->type = INSTR_ALU_OR_I;
3028         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3029         instr->alu.dst.n_bits = fdst->n_bits;
3030         instr->alu.dst.offset = fdst->offset / 8;
3031         instr->alu.src_val = src_val;
3032         return 0;
3033 }
3034
3035 static int
3036 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3037                         struct action *action,
3038                         char **tokens,
3039                         int n_tokens,
3040                         struct instruction *instr,
3041                         struct instruction_data *data __rte_unused)
3042 {
3043         char *dst = tokens[1], *src = tokens[2];
3044         struct field *fdst, *fsrc;
3045         uint64_t src_val;
3046         uint32_t dst_struct_id = 0, src_struct_id = 0;
3047
3048         CHECK(n_tokens == 3, EINVAL);
3049
3050         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3051         CHECK(fdst, EINVAL);
3052         CHECK(!fdst->var_size, EINVAL);
3053
3054         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3055         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3056         if (fsrc) {
3057                 CHECK(!fsrc->var_size, EINVAL);
3058
3059                 instr->type = INSTR_ALU_XOR;
3060                 if (dst[0] != 'h' && src[0] == 'h')
3061                         instr->type = INSTR_ALU_XOR_MH;
3062                 if (dst[0] == 'h' && src[0] != 'h')
3063                         instr->type = INSTR_ALU_XOR_HM;
3064                 if (dst[0] == 'h' && src[0] == 'h')
3065                         instr->type = INSTR_ALU_XOR_HH;
3066
3067                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3068                 instr->alu.dst.n_bits = fdst->n_bits;
3069                 instr->alu.dst.offset = fdst->offset / 8;
3070                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3071                 instr->alu.src.n_bits = fsrc->n_bits;
3072                 instr->alu.src.offset = fsrc->offset / 8;
3073                 return 0;
3074         }
3075
3076         /* XOR_I. */
3077         src_val = strtoull(src, &src, 0);
3078         CHECK(!src[0], EINVAL);
3079
3080         if (dst[0] == 'h')
3081                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3082
3083         instr->type = INSTR_ALU_XOR_I;
3084         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3085         instr->alu.dst.n_bits = fdst->n_bits;
3086         instr->alu.dst.offset = fdst->offset / 8;
3087         instr->alu.src_val = src_val;
3088         return 0;
3089 }
3090
3091 static inline void
3092 instr_alu_add_exec(struct rte_swx_pipeline *p)
3093 {
3094         struct thread *t = &p->threads[p->thread_id];
3095         struct instruction *ip = t->ip;
3096
3097         TRACE("[Thread %2u] add\n", p->thread_id);
3098
3099         /* Structs. */
3100         ALU(t, ip, +);
3101
3102         /* Thread. */
3103         thread_ip_inc(p);
3104 }
3105
3106 static inline void
3107 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3108 {
3109         struct thread *t = &p->threads[p->thread_id];
3110         struct instruction *ip = t->ip;
3111
3112         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3113
3114         /* Structs. */
3115         ALU_MH(t, ip, +);
3116
3117         /* Thread. */
3118         thread_ip_inc(p);
3119 }
3120
3121 static inline void
3122 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3123 {
3124         struct thread *t = &p->threads[p->thread_id];
3125         struct instruction *ip = t->ip;
3126
3127         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3128
3129         /* Structs. */
3130         ALU_HM(t, ip, +);
3131
3132         /* Thread. */
3133         thread_ip_inc(p);
3134 }
3135
3136 static inline void
3137 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3138 {
3139         struct thread *t = &p->threads[p->thread_id];
3140         struct instruction *ip = t->ip;
3141
3142         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3143
3144         /* Structs. */
3145         ALU_HH(t, ip, +);
3146
3147         /* Thread. */
3148         thread_ip_inc(p);
3149 }
3150
3151 static inline void
3152 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3153 {
3154         struct thread *t = &p->threads[p->thread_id];
3155         struct instruction *ip = t->ip;
3156
3157         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3158
3159         /* Structs. */
3160         ALU_MI(t, ip, +);
3161
3162         /* Thread. */
3163         thread_ip_inc(p);
3164 }
3165
3166 static inline void
3167 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3168 {
3169         struct thread *t = &p->threads[p->thread_id];
3170         struct instruction *ip = t->ip;
3171
3172         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3173
3174         /* Structs. */
3175         ALU_HI(t, ip, +);
3176
3177         /* Thread. */
3178         thread_ip_inc(p);
3179 }
3180
3181 static inline void
3182 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3183 {
3184         struct thread *t = &p->threads[p->thread_id];
3185         struct instruction *ip = t->ip;
3186
3187         TRACE("[Thread %2u] sub\n", p->thread_id);
3188
3189         /* Structs. */
3190         ALU(t, ip, -);
3191
3192         /* Thread. */
3193         thread_ip_inc(p);
3194 }
3195
3196 static inline void
3197 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3198 {
3199         struct thread *t = &p->threads[p->thread_id];
3200         struct instruction *ip = t->ip;
3201
3202         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3203
3204         /* Structs. */
3205         ALU_MH(t, ip, -);
3206
3207         /* Thread. */
3208         thread_ip_inc(p);
3209 }
3210
3211 static inline void
3212 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3213 {
3214         struct thread *t = &p->threads[p->thread_id];
3215         struct instruction *ip = t->ip;
3216
3217         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3218
3219         /* Structs. */
3220         ALU_HM(t, ip, -);
3221
3222         /* Thread. */
3223         thread_ip_inc(p);
3224 }
3225
3226 static inline void
3227 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3228 {
3229         struct thread *t = &p->threads[p->thread_id];
3230         struct instruction *ip = t->ip;
3231
3232         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3233
3234         /* Structs. */
3235         ALU_HH(t, ip, -);
3236
3237         /* Thread. */
3238         thread_ip_inc(p);
3239 }
3240
3241 static inline void
3242 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3243 {
3244         struct thread *t = &p->threads[p->thread_id];
3245         struct instruction *ip = t->ip;
3246
3247         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3248
3249         /* Structs. */
3250         ALU_MI(t, ip, -);
3251
3252         /* Thread. */
3253         thread_ip_inc(p);
3254 }
3255
3256 static inline void
3257 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3258 {
3259         struct thread *t = &p->threads[p->thread_id];
3260         struct instruction *ip = t->ip;
3261
3262         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3263
3264         /* Structs. */
3265         ALU_HI(t, ip, -);
3266
3267         /* Thread. */
3268         thread_ip_inc(p);
3269 }
3270
3271 static inline void
3272 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3273 {
3274         struct thread *t = &p->threads[p->thread_id];
3275         struct instruction *ip = t->ip;
3276
3277         TRACE("[Thread %2u] shl\n", p->thread_id);
3278
3279         /* Structs. */
3280         ALU(t, ip, <<);
3281
3282         /* Thread. */
3283         thread_ip_inc(p);
3284 }
3285
3286 static inline void
3287 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3288 {
3289         struct thread *t = &p->threads[p->thread_id];
3290         struct instruction *ip = t->ip;
3291
3292         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3293
3294         /* Structs. */
3295         ALU_MH(t, ip, <<);
3296
3297         /* Thread. */
3298         thread_ip_inc(p);
3299 }
3300
3301 static inline void
3302 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3303 {
3304         struct thread *t = &p->threads[p->thread_id];
3305         struct instruction *ip = t->ip;
3306
3307         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3308
3309         /* Structs. */
3310         ALU_HM(t, ip, <<);
3311
3312         /* Thread. */
3313         thread_ip_inc(p);
3314 }
3315
3316 static inline void
3317 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3318 {
3319         struct thread *t = &p->threads[p->thread_id];
3320         struct instruction *ip = t->ip;
3321
3322         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3323
3324         /* Structs. */
3325         ALU_HH(t, ip, <<);
3326
3327         /* Thread. */
3328         thread_ip_inc(p);
3329 }
3330
3331 static inline void
3332 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3333 {
3334         struct thread *t = &p->threads[p->thread_id];
3335         struct instruction *ip = t->ip;
3336
3337         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3338
3339         /* Structs. */
3340         ALU_MI(t, ip, <<);
3341
3342         /* Thread. */
3343         thread_ip_inc(p);
3344 }
3345
3346 static inline void
3347 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3348 {
3349         struct thread *t = &p->threads[p->thread_id];
3350         struct instruction *ip = t->ip;
3351
3352         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3353
3354         /* Structs. */
3355         ALU_HI(t, ip, <<);
3356
3357         /* Thread. */
3358         thread_ip_inc(p);
3359 }
3360
3361 static inline void
3362 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3363 {
3364         struct thread *t = &p->threads[p->thread_id];
3365         struct instruction *ip = t->ip;
3366
3367         TRACE("[Thread %2u] shr\n", p->thread_id);
3368
3369         /* Structs. */
3370         ALU(t, ip, >>);
3371
3372         /* Thread. */
3373         thread_ip_inc(p);
3374 }
3375
3376 static inline void
3377 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3378 {
3379         struct thread *t = &p->threads[p->thread_id];
3380         struct instruction *ip = t->ip;
3381
3382         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3383
3384         /* Structs. */
3385         ALU_MH(t, ip, >>);
3386
3387         /* Thread. */
3388         thread_ip_inc(p);
3389 }
3390
3391 static inline void
3392 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3393 {
3394         struct thread *t = &p->threads[p->thread_id];
3395         struct instruction *ip = t->ip;
3396
3397         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3398
3399         /* Structs. */
3400         ALU_HM(t, ip, >>);
3401
3402         /* Thread. */
3403         thread_ip_inc(p);
3404 }
3405
3406 static inline void
3407 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3408 {
3409         struct thread *t = &p->threads[p->thread_id];
3410         struct instruction *ip = t->ip;
3411
3412         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3413
3414         /* Structs. */
3415         ALU_HH(t, ip, >>);
3416
3417         /* Thread. */
3418         thread_ip_inc(p);
3419 }
3420
3421 static inline void
3422 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3423 {
3424         struct thread *t = &p->threads[p->thread_id];
3425         struct instruction *ip = t->ip;
3426
3427         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3428
3429         /* Structs. */
3430         ALU_MI(t, ip, >>);
3431
3432         /* Thread. */
3433         thread_ip_inc(p);
3434 }
3435
3436 static inline void
3437 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3438 {
3439         struct thread *t = &p->threads[p->thread_id];
3440         struct instruction *ip = t->ip;
3441
3442         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3443
3444         /* Structs. */
3445         ALU_HI(t, ip, >>);
3446
3447         /* Thread. */
3448         thread_ip_inc(p);
3449 }
3450
3451 static inline void
3452 instr_alu_and_exec(struct rte_swx_pipeline *p)
3453 {
3454         struct thread *t = &p->threads[p->thread_id];
3455         struct instruction *ip = t->ip;
3456
3457         TRACE("[Thread %2u] and\n", p->thread_id);
3458
3459         /* Structs. */
3460         ALU(t, ip, &);
3461
3462         /* Thread. */
3463         thread_ip_inc(p);
3464 }
3465
3466 static inline void
3467 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3468 {
3469         struct thread *t = &p->threads[p->thread_id];
3470         struct instruction *ip = t->ip;
3471
3472         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3473
3474         /* Structs. */
3475         ALU_MH(t, ip, &);
3476
3477         /* Thread. */
3478         thread_ip_inc(p);
3479 }
3480
3481 static inline void
3482 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3483 {
3484         struct thread *t = &p->threads[p->thread_id];
3485         struct instruction *ip = t->ip;
3486
3487         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3488
3489         /* Structs. */
3490         ALU_HM_FAST(t, ip, &);
3491
3492         /* Thread. */
3493         thread_ip_inc(p);
3494 }
3495
3496 static inline void
3497 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3498 {
3499         struct thread *t = &p->threads[p->thread_id];
3500         struct instruction *ip = t->ip;
3501
3502         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3503
3504         /* Structs. */
3505         ALU_HH_FAST(t, ip, &);
3506
3507         /* Thread. */
3508         thread_ip_inc(p);
3509 }
3510
3511 static inline void
3512 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3513 {
3514         struct thread *t = &p->threads[p->thread_id];
3515         struct instruction *ip = t->ip;
3516
3517         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3518
3519         /* Structs. */
3520         ALU_I(t, ip, &);
3521
3522         /* Thread. */
3523         thread_ip_inc(p);
3524 }
3525
3526 static inline void
3527 instr_alu_or_exec(struct rte_swx_pipeline *p)
3528 {
3529         struct thread *t = &p->threads[p->thread_id];
3530         struct instruction *ip = t->ip;
3531
3532         TRACE("[Thread %2u] or\n", p->thread_id);
3533
3534         /* Structs. */
3535         ALU(t, ip, |);
3536
3537         /* Thread. */
3538         thread_ip_inc(p);
3539 }
3540
3541 static inline void
3542 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3543 {
3544         struct thread *t = &p->threads[p->thread_id];
3545         struct instruction *ip = t->ip;
3546
3547         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3548
3549         /* Structs. */
3550         ALU_MH(t, ip, |);
3551
3552         /* Thread. */
3553         thread_ip_inc(p);
3554 }
3555
3556 static inline void
3557 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3558 {
3559         struct thread *t = &p->threads[p->thread_id];
3560         struct instruction *ip = t->ip;
3561
3562         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3563
3564         /* Structs. */
3565         ALU_HM_FAST(t, ip, |);
3566
3567         /* Thread. */
3568         thread_ip_inc(p);
3569 }
3570
3571 static inline void
3572 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3573 {
3574         struct thread *t = &p->threads[p->thread_id];
3575         struct instruction *ip = t->ip;
3576
3577         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3578
3579         /* Structs. */
3580         ALU_HH_FAST(t, ip, |);
3581
3582         /* Thread. */
3583         thread_ip_inc(p);
3584 }
3585
3586 static inline void
3587 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3588 {
3589         struct thread *t = &p->threads[p->thread_id];
3590         struct instruction *ip = t->ip;
3591
3592         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3593
3594         /* Structs. */
3595         ALU_I(t, ip, |);
3596
3597         /* Thread. */
3598         thread_ip_inc(p);
3599 }
3600
3601 static inline void
3602 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3603 {
3604         struct thread *t = &p->threads[p->thread_id];
3605         struct instruction *ip = t->ip;
3606
3607         TRACE("[Thread %2u] xor\n", p->thread_id);
3608
3609         /* Structs. */
3610         ALU(t, ip, ^);
3611
3612         /* Thread. */
3613         thread_ip_inc(p);
3614 }
3615
3616 static inline void
3617 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3618 {
3619         struct thread *t = &p->threads[p->thread_id];
3620         struct instruction *ip = t->ip;
3621
3622         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3623
3624         /* Structs. */
3625         ALU_MH(t, ip, ^);
3626
3627         /* Thread. */
3628         thread_ip_inc(p);
3629 }
3630
3631 static inline void
3632 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3633 {
3634         struct thread *t = &p->threads[p->thread_id];
3635         struct instruction *ip = t->ip;
3636
3637         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3638
3639         /* Structs. */
3640         ALU_HM_FAST(t, ip, ^);
3641
3642         /* Thread. */
3643         thread_ip_inc(p);
3644 }
3645
3646 static inline void
3647 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3648 {
3649         struct thread *t = &p->threads[p->thread_id];
3650         struct instruction *ip = t->ip;
3651
3652         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3653
3654         /* Structs. */
3655         ALU_HH_FAST(t, ip, ^);
3656
3657         /* Thread. */
3658         thread_ip_inc(p);
3659 }
3660
3661 static inline void
3662 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3663 {
3664         struct thread *t = &p->threads[p->thread_id];
3665         struct instruction *ip = t->ip;
3666
3667         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3668
3669         /* Structs. */
3670         ALU_I(t, ip, ^);
3671
3672         /* Thread. */
3673         thread_ip_inc(p);
3674 }
3675
3676 static inline void
3677 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3678 {
3679         struct thread *t = &p->threads[p->thread_id];
3680         struct instruction *ip = t->ip;
3681         uint8_t *dst_struct, *src_struct;
3682         uint16_t *dst16_ptr, dst;
3683         uint64_t *src64_ptr, src64, src64_mask, src;
3684         uint64_t r;
3685
3686         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3687
3688         /* Structs. */
3689         dst_struct = t->structs[ip->alu.dst.struct_id];
3690         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3691         dst = *dst16_ptr;
3692
3693         src_struct = t->structs[ip->alu.src.struct_id];
3694         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3695         src64 = *src64_ptr;
3696         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3697         src = src64 & src64_mask;
3698
3699         r = dst;
3700         r = ~r & 0xFFFF;
3701
3702         /* The first input (r) is a 16-bit number. The second and the third
3703          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3704          * three numbers (output r) is a 34-bit number.
3705          */
3706         r += (src >> 32) + (src & 0xFFFFFFFF);
3707
3708         /* The first input is a 16-bit number. The second input is an 18-bit
3709          * number. In the worst case scenario, the sum of the two numbers is a
3710          * 19-bit number.
3711          */
3712         r = (r & 0xFFFF) + (r >> 16);
3713
3714         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3715          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3716          */
3717         r = (r & 0xFFFF) + (r >> 16);
3718
3719         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3720          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3721          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3722          * therefore the output r is always a 16-bit number.
3723          */
3724         r = (r & 0xFFFF) + (r >> 16);
3725
3726         r = ~r & 0xFFFF;
3727         r = r ? r : 0xFFFF;
3728
3729         *dst16_ptr = (uint16_t)r;
3730
3731         /* Thread. */
3732         thread_ip_inc(p);
3733 }
3734
3735 static inline void
3736 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3737 {
3738         struct thread *t = &p->threads[p->thread_id];
3739         struct instruction *ip = t->ip;
3740         uint8_t *dst_struct, *src_struct;
3741         uint16_t *dst16_ptr, dst;
3742         uint64_t *src64_ptr, src64, src64_mask, src;
3743         uint64_t r;
3744
3745         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3746
3747         /* Structs. */
3748         dst_struct = t->structs[ip->alu.dst.struct_id];
3749         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3750         dst = *dst16_ptr;
3751
3752         src_struct = t->structs[ip->alu.src.struct_id];
3753         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3754         src64 = *src64_ptr;
3755         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3756         src = src64 & src64_mask;
3757
3758         r = dst;
3759         r = ~r & 0xFFFF;
3760
3761         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3762          * the following sequence of operations in 2's complement arithmetic:
3763          *    a '- b = (a - b) % 0xFFFF.
3764          *
3765          * In order to prevent an underflow for the below subtraction, in which
3766          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3767          * minuend), we first add a multiple of the 0xFFFF modulus to the
3768          * minuend. The number we add to the minuend needs to be a 34-bit number
3769          * or higher, so for readability reasons we picked the 36-bit multiple.
3770          * We are effectively turning the 16-bit minuend into a 36-bit number:
3771          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3772          */
3773         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3774
3775         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3776          * result (the output r) is a 36-bit number.
3777          */
3778         r -= (src >> 32) + (src & 0xFFFFFFFF);
3779
3780         /* The first input is a 16-bit number. The second input is a 20-bit
3781          * number. Their sum is a 21-bit number.
3782          */
3783         r = (r & 0xFFFF) + (r >> 16);
3784
3785         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3786          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3787          */
3788         r = (r & 0xFFFF) + (r >> 16);
3789
3790         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3791          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3792          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3793          * generated, therefore the output r is always a 16-bit number.
3794          */
3795         r = (r & 0xFFFF) + (r >> 16);
3796
3797         r = ~r & 0xFFFF;
3798         r = r ? r : 0xFFFF;
3799
3800         *dst16_ptr = (uint16_t)r;
3801
3802         /* Thread. */
3803         thread_ip_inc(p);
3804 }
3805
3806 static inline void
3807 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3808 {
3809         struct thread *t = &p->threads[p->thread_id];
3810         struct instruction *ip = t->ip;
3811         uint8_t *dst_struct, *src_struct;
3812         uint16_t *dst16_ptr;
3813         uint32_t *src32_ptr;
3814         uint64_t r0, r1;
3815
3816         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3817
3818         /* Structs. */
3819         dst_struct = t->structs[ip->alu.dst.struct_id];
3820         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3821
3822         src_struct = t->structs[ip->alu.src.struct_id];
3823         src32_ptr = (uint32_t *)&src_struct[0];
3824
3825         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3826         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3827         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3828         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3829         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3830
3831         /* The first input is a 16-bit number. The second input is a 19-bit
3832          * number. Their sum is a 20-bit number.
3833          */
3834         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3835
3836         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3837          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3838          */
3839         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3840
3841         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3842          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3843          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3844          * generated, therefore the output r is always a 16-bit number.
3845          */
3846         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3847
3848         r0 = ~r0 & 0xFFFF;
3849         r0 = r0 ? r0 : 0xFFFF;
3850
3851         *dst16_ptr = (uint16_t)r0;
3852
3853         /* Thread. */
3854         thread_ip_inc(p);
3855 }
3856
3857 static inline void
3858 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3859 {
3860         struct thread *t = &p->threads[p->thread_id];
3861         struct instruction *ip = t->ip;
3862         uint8_t *dst_struct, *src_struct;
3863         uint16_t *dst16_ptr;
3864         uint32_t *src32_ptr;
3865         uint64_t r = 0;
3866         uint32_t i;
3867
3868         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3869
3870         /* Structs. */
3871         dst_struct = t->structs[ip->alu.dst.struct_id];
3872         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3873
3874         src_struct = t->structs[ip->alu.src.struct_id];
3875         src32_ptr = (uint32_t *)&src_struct[0];
3876
3877         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3878          * Therefore, in the worst case scenario, a 35-bit number is added to a
3879          * 16-bit number (the input r), so the output r is 36-bit number.
3880          */
3881         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3882                 r += *src32_ptr;
3883
3884         /* The first input is a 16-bit number. The second input is a 20-bit
3885          * number. Their sum is a 21-bit number.
3886          */
3887         r = (r & 0xFFFF) + (r >> 16);
3888
3889         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3890          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3891          */
3892         r = (r & 0xFFFF) + (r >> 16);
3893
3894         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3895          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3896          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3897          * generated, therefore the output r is always a 16-bit number.
3898          */
3899         r = (r & 0xFFFF) + (r >> 16);
3900
3901         r = ~r & 0xFFFF;
3902         r = r ? r : 0xFFFF;
3903
3904         *dst16_ptr = (uint16_t)r;
3905
3906         /* Thread. */
3907         thread_ip_inc(p);
3908 }
3909
3910 /*
3911  * Register array.
3912  */
3913 static struct regarray *
3914 regarray_find(struct rte_swx_pipeline *p, const char *name);
3915
3916 static int
3917 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3918                       struct action *action,
3919                       char **tokens,
3920                       int n_tokens,
3921                       struct instruction *instr,
3922                       struct instruction_data *data __rte_unused)
3923 {
3924         char *regarray = tokens[1], *idx = tokens[2];
3925         struct regarray *r;
3926         struct field *fidx;
3927         uint32_t idx_struct_id, idx_val;
3928
3929         CHECK(n_tokens == 3, EINVAL);
3930
3931         r = regarray_find(p, regarray);
3932         CHECK(r, EINVAL);
3933
3934         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3935         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3936         if (fidx) {
3937                 CHECK(!fidx->var_size, EINVAL);
3938
3939                 instr->type = INSTR_REGPREFETCH_RM;
3940                 if (idx[0] == 'h')
3941                         instr->type = INSTR_REGPREFETCH_RH;
3942
3943                 instr->regarray.regarray_id = r->id;
3944                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3945                 instr->regarray.idx.n_bits = fidx->n_bits;
3946                 instr->regarray.idx.offset = fidx->offset / 8;
3947                 instr->regarray.dstsrc_val = 0; /* Unused. */
3948                 return 0;
3949         }
3950
3951         /* REGPREFETCH_RI. */
3952         idx_val = strtoul(idx, &idx, 0);
3953         CHECK(!idx[0], EINVAL);
3954
3955         instr->type = INSTR_REGPREFETCH_RI;
3956         instr->regarray.regarray_id = r->id;
3957         instr->regarray.idx_val = idx_val;
3958         instr->regarray.dstsrc_val = 0; /* Unused. */
3959         return 0;
3960 }
3961
3962 static int
3963 instr_regrd_translate(struct rte_swx_pipeline *p,
3964                       struct action *action,
3965                       char **tokens,
3966                       int n_tokens,
3967                       struct instruction *instr,
3968                       struct instruction_data *data __rte_unused)
3969 {
3970         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3971         struct regarray *r;
3972         struct field *fdst, *fidx;
3973         uint32_t dst_struct_id, idx_struct_id, idx_val;
3974
3975         CHECK(n_tokens == 4, EINVAL);
3976
3977         r = regarray_find(p, regarray);
3978         CHECK(r, EINVAL);
3979
3980         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3981         CHECK(fdst, EINVAL);
3982         CHECK(!fdst->var_size, EINVAL);
3983
3984         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3985         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3986         if (fidx) {
3987                 CHECK(!fidx->var_size, EINVAL);
3988
3989                 instr->type = INSTR_REGRD_MRM;
3990                 if (dst[0] == 'h' && idx[0] != 'h')
3991                         instr->type = INSTR_REGRD_HRM;
3992                 if (dst[0] != 'h' && idx[0] == 'h')
3993                         instr->type = INSTR_REGRD_MRH;
3994                 if (dst[0] == 'h' && idx[0] == 'h')
3995                         instr->type = INSTR_REGRD_HRH;
3996
3997                 instr->regarray.regarray_id = r->id;
3998                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3999                 instr->regarray.idx.n_bits = fidx->n_bits;
4000                 instr->regarray.idx.offset = fidx->offset / 8;
4001                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4002                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4003                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4004                 return 0;
4005         }
4006
4007         /* REGRD_MRI, REGRD_HRI. */
4008         idx_val = strtoul(idx, &idx, 0);
4009         CHECK(!idx[0], EINVAL);
4010
4011         instr->type = INSTR_REGRD_MRI;
4012         if (dst[0] == 'h')
4013                 instr->type = INSTR_REGRD_HRI;
4014
4015         instr->regarray.regarray_id = r->id;
4016         instr->regarray.idx_val = idx_val;
4017         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4018         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4019         instr->regarray.dstsrc.offset = fdst->offset / 8;
4020         return 0;
4021 }
4022
4023 static int
4024 instr_regwr_translate(struct rte_swx_pipeline *p,
4025                       struct action *action,
4026                       char **tokens,
4027                       int n_tokens,
4028                       struct instruction *instr,
4029                       struct instruction_data *data __rte_unused)
4030 {
4031         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4032         struct regarray *r;
4033         struct field *fidx, *fsrc;
4034         uint64_t src_val;
4035         uint32_t idx_struct_id, idx_val, src_struct_id;
4036
4037         CHECK(n_tokens == 4, EINVAL);
4038
4039         r = regarray_find(p, regarray);
4040         CHECK(r, EINVAL);
4041
4042         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4043         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4044         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4045         if (fidx && fsrc) {
4046                 CHECK(!fidx->var_size, EINVAL);
4047                 CHECK(!fsrc->var_size, EINVAL);
4048
4049                 instr->type = INSTR_REGWR_RMM;
4050                 if (idx[0] == 'h' && src[0] != 'h')
4051                         instr->type = INSTR_REGWR_RHM;
4052                 if (idx[0] != 'h' && src[0] == 'h')
4053                         instr->type = INSTR_REGWR_RMH;
4054                 if (idx[0] == 'h' && src[0] == 'h')
4055                         instr->type = INSTR_REGWR_RHH;
4056
4057                 instr->regarray.regarray_id = r->id;
4058                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4059                 instr->regarray.idx.n_bits = fidx->n_bits;
4060                 instr->regarray.idx.offset = fidx->offset / 8;
4061                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4062                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4063                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4064                 return 0;
4065         }
4066
4067         /* REGWR_RHI, REGWR_RMI. */
4068         if (fidx && !fsrc) {
4069                 CHECK(!fidx->var_size, EINVAL);
4070
4071                 src_val = strtoull(src, &src, 0);
4072                 CHECK(!src[0], EINVAL);
4073
4074                 instr->type = INSTR_REGWR_RMI;
4075                 if (idx[0] == 'h')
4076                         instr->type = INSTR_REGWR_RHI;
4077
4078                 instr->regarray.regarray_id = r->id;
4079                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4080                 instr->regarray.idx.n_bits = fidx->n_bits;
4081                 instr->regarray.idx.offset = fidx->offset / 8;
4082                 instr->regarray.dstsrc_val = src_val;
4083                 return 0;
4084         }
4085
4086         /* REGWR_RIH, REGWR_RIM. */
4087         if (!fidx && fsrc) {
4088                 idx_val = strtoul(idx, &idx, 0);
4089                 CHECK(!idx[0], EINVAL);
4090
4091                 CHECK(!fsrc->var_size, EINVAL);
4092
4093                 instr->type = INSTR_REGWR_RIM;
4094                 if (src[0] == 'h')
4095                         instr->type = INSTR_REGWR_RIH;
4096
4097                 instr->regarray.regarray_id = r->id;
4098                 instr->regarray.idx_val = idx_val;
4099                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4100                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4101                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4102                 return 0;
4103         }
4104
4105         /* REGWR_RII. */
4106         src_val = strtoull(src, &src, 0);
4107         CHECK(!src[0], EINVAL);
4108
4109         idx_val = strtoul(idx, &idx, 0);
4110         CHECK(!idx[0], EINVAL);
4111
4112         instr->type = INSTR_REGWR_RII;
4113         instr->regarray.idx_val = idx_val;
4114         instr->regarray.dstsrc_val = src_val;
4115
4116         return 0;
4117 }
4118
4119 static int
4120 instr_regadd_translate(struct rte_swx_pipeline *p,
4121                        struct action *action,
4122                        char **tokens,
4123                        int n_tokens,
4124                        struct instruction *instr,
4125                        struct instruction_data *data __rte_unused)
4126 {
4127         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4128         struct regarray *r;
4129         struct field *fidx, *fsrc;
4130         uint64_t src_val;
4131         uint32_t idx_struct_id, idx_val, src_struct_id;
4132
4133         CHECK(n_tokens == 4, EINVAL);
4134
4135         r = regarray_find(p, regarray);
4136         CHECK(r, EINVAL);
4137
4138         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4139         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4140         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4141         if (fidx && fsrc) {
4142                 CHECK(!fidx->var_size, EINVAL);
4143                 CHECK(!fsrc->var_size, EINVAL);
4144
4145                 instr->type = INSTR_REGADD_RMM;
4146                 if (idx[0] == 'h' && src[0] != 'h')
4147                         instr->type = INSTR_REGADD_RHM;
4148                 if (idx[0] != 'h' && src[0] == 'h')
4149                         instr->type = INSTR_REGADD_RMH;
4150                 if (idx[0] == 'h' && src[0] == 'h')
4151                         instr->type = INSTR_REGADD_RHH;
4152
4153                 instr->regarray.regarray_id = r->id;
4154                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4155                 instr->regarray.idx.n_bits = fidx->n_bits;
4156                 instr->regarray.idx.offset = fidx->offset / 8;
4157                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4158                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4159                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4160                 return 0;
4161         }
4162
4163         /* REGADD_RHI, REGADD_RMI. */
4164         if (fidx && !fsrc) {
4165                 CHECK(!fidx->var_size, EINVAL);
4166
4167                 src_val = strtoull(src, &src, 0);
4168                 CHECK(!src[0], EINVAL);
4169
4170                 instr->type = INSTR_REGADD_RMI;
4171                 if (idx[0] == 'h')
4172                         instr->type = INSTR_REGADD_RHI;
4173
4174                 instr->regarray.regarray_id = r->id;
4175                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4176                 instr->regarray.idx.n_bits = fidx->n_bits;
4177                 instr->regarray.idx.offset = fidx->offset / 8;
4178                 instr->regarray.dstsrc_val = src_val;
4179                 return 0;
4180         }
4181
4182         /* REGADD_RIH, REGADD_RIM. */
4183         if (!fidx && fsrc) {
4184                 idx_val = strtoul(idx, &idx, 0);
4185                 CHECK(!idx[0], EINVAL);
4186
4187                 CHECK(!fsrc->var_size, EINVAL);
4188
4189                 instr->type = INSTR_REGADD_RIM;
4190                 if (src[0] == 'h')
4191                         instr->type = INSTR_REGADD_RIH;
4192
4193                 instr->regarray.regarray_id = r->id;
4194                 instr->regarray.idx_val = idx_val;
4195                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4196                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4197                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4198                 return 0;
4199         }
4200
4201         /* REGADD_RII. */
4202         src_val = strtoull(src, &src, 0);
4203         CHECK(!src[0], EINVAL);
4204
4205         idx_val = strtoul(idx, &idx, 0);
4206         CHECK(!idx[0], EINVAL);
4207
4208         instr->type = INSTR_REGADD_RII;
4209         instr->regarray.idx_val = idx_val;
4210         instr->regarray.dstsrc_val = src_val;
4211         return 0;
4212 }
4213
4214 static inline uint64_t *
4215 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4216 {
4217         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4218         return r->regarray;
4219 }
4220
4221 static inline uint64_t
4222 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4223 {
4224         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4225
4226         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4227         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4228         uint64_t idx64 = *idx64_ptr;
4229         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4230         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4231
4232         return idx;
4233 }
4234
4235 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4236
4237 static inline uint64_t
4238 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4239 {
4240         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4241
4242         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4243         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4244         uint64_t idx64 = *idx64_ptr;
4245         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4246
4247         return idx;
4248 }
4249
4250 #else
4251
4252 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4253
4254 #endif
4255
4256 static inline uint64_t
4257 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4258 {
4259         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4260
4261         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4262
4263         return idx;
4264 }
4265
4266 static inline uint64_t
4267 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4268 {
4269         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4270         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4271         uint64_t src64 = *src64_ptr;
4272         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4273         uint64_t src = src64 & src64_mask;
4274
4275         return src;
4276 }
4277
4278 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4279
4280 static inline uint64_t
4281 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4282 {
4283         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4284         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4285         uint64_t src64 = *src64_ptr;
4286         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4287
4288         return src;
4289 }
4290
4291 #else
4292
4293 #define instr_regarray_src_nbo instr_regarray_src_hbo
4294
4295 #endif
4296
4297 static inline void
4298 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4299 {
4300         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4301         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4302         uint64_t dst64 = *dst64_ptr;
4303         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4304
4305         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4306
4307 }
4308
4309 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4310
4311 static inline void
4312 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4313 {
4314         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4315         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4316         uint64_t dst64 = *dst64_ptr;
4317         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4318
4319         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4320         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4321 }
4322
4323 #else
4324
4325 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4326
4327 #endif
4328
4329 static inline void
4330 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4331 {
4332         struct thread *t = &p->threads[p->thread_id];
4333         struct instruction *ip = t->ip;
4334         uint64_t *regarray, idx;
4335
4336         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4337
4338         /* Structs. */
4339         regarray = instr_regarray_regarray(p, ip);
4340         idx = instr_regarray_idx_nbo(p, t, ip);
4341         rte_prefetch0(&regarray[idx]);
4342
4343         /* Thread. */
4344         thread_ip_inc(p);
4345 }
4346
4347 static inline void
4348 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4349 {
4350         struct thread *t = &p->threads[p->thread_id];
4351         struct instruction *ip = t->ip;
4352         uint64_t *regarray, idx;
4353
4354         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4355
4356         /* Structs. */
4357         regarray = instr_regarray_regarray(p, ip);
4358         idx = instr_regarray_idx_hbo(p, t, ip);
4359         rte_prefetch0(&regarray[idx]);
4360
4361         /* Thread. */
4362         thread_ip_inc(p);
4363 }
4364
4365 static inline void
4366 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4367 {
4368         struct thread *t = &p->threads[p->thread_id];
4369         struct instruction *ip = t->ip;
4370         uint64_t *regarray, idx;
4371
4372         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4373
4374         /* Structs. */
4375         regarray = instr_regarray_regarray(p, ip);
4376         idx = instr_regarray_idx_imm(p, ip);
4377         rte_prefetch0(&regarray[idx]);
4378
4379         /* Thread. */
4380         thread_ip_inc(p);
4381 }
4382
4383 static inline void
4384 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4385 {
4386         struct thread *t = &p->threads[p->thread_id];
4387         struct instruction *ip = t->ip;
4388         uint64_t *regarray, idx;
4389
4390         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4391
4392         /* Structs. */
4393         regarray = instr_regarray_regarray(p, ip);
4394         idx = instr_regarray_idx_nbo(p, t, ip);
4395         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4396
4397         /* Thread. */
4398         thread_ip_inc(p);
4399 }
4400
4401 static inline void
4402 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4403 {
4404         struct thread *t = &p->threads[p->thread_id];
4405         struct instruction *ip = t->ip;
4406         uint64_t *regarray, idx;
4407
4408         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4409
4410         /* Structs. */
4411         regarray = instr_regarray_regarray(p, ip);
4412         idx = instr_regarray_idx_hbo(p, t, ip);
4413         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4414
4415         /* Thread. */
4416         thread_ip_inc(p);
4417 }
4418
4419 static inline void
4420 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4421 {
4422         struct thread *t = &p->threads[p->thread_id];
4423         struct instruction *ip = t->ip;
4424         uint64_t *regarray, idx;
4425
4426         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4427
4428         /* Structs. */
4429         regarray = instr_regarray_regarray(p, ip);
4430         idx = instr_regarray_idx_nbo(p, t, ip);
4431         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4432
4433         /* Thread. */
4434         thread_ip_inc(p);
4435 }
4436
4437 static inline void
4438 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4439 {
4440         struct thread *t = &p->threads[p->thread_id];
4441         struct instruction *ip = t->ip;
4442         uint64_t *regarray, idx;
4443
4444         /* Structs. */
4445         regarray = instr_regarray_regarray(p, ip);
4446         idx = instr_regarray_idx_hbo(p, t, ip);
4447         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4448
4449         /* Thread. */
4450         thread_ip_inc(p);
4451 }
4452
4453 static inline void
4454 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4455 {
4456         struct thread *t = &p->threads[p->thread_id];
4457         struct instruction *ip = t->ip;
4458         uint64_t *regarray, idx;
4459
4460         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4461
4462         /* Structs. */
4463         regarray = instr_regarray_regarray(p, ip);
4464         idx = instr_regarray_idx_imm(p, ip);
4465         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4466
4467         /* Thread. */
4468         thread_ip_inc(p);
4469 }
4470
4471 static inline void
4472 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4473 {
4474         struct thread *t = &p->threads[p->thread_id];
4475         struct instruction *ip = t->ip;
4476         uint64_t *regarray, idx;
4477
4478         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4479
4480         /* Structs. */
4481         regarray = instr_regarray_regarray(p, ip);
4482         idx = instr_regarray_idx_imm(p, ip);
4483         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4484
4485         /* Thread. */
4486         thread_ip_inc(p);
4487 }
4488
4489 static inline void
4490 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4491 {
4492         struct thread *t = &p->threads[p->thread_id];
4493         struct instruction *ip = t->ip;
4494         uint64_t *regarray, idx, src;
4495
4496         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4497
4498         /* Structs. */
4499         regarray = instr_regarray_regarray(p, ip);
4500         idx = instr_regarray_idx_nbo(p, t, ip);
4501         src = instr_regarray_src_nbo(t, ip);
4502         regarray[idx] = src;
4503
4504         /* Thread. */
4505         thread_ip_inc(p);
4506 }
4507
4508 static inline void
4509 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4510 {
4511         struct thread *t = &p->threads[p->thread_id];
4512         struct instruction *ip = t->ip;
4513         uint64_t *regarray, idx, src;
4514
4515         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4516
4517         /* Structs. */
4518         regarray = instr_regarray_regarray(p, ip);
4519         idx = instr_regarray_idx_nbo(p, t, ip);
4520         src = instr_regarray_src_hbo(t, ip);
4521         regarray[idx] = src;
4522
4523         /* Thread. */
4524         thread_ip_inc(p);
4525 }
4526
4527 static inline void
4528 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4529 {
4530         struct thread *t = &p->threads[p->thread_id];
4531         struct instruction *ip = t->ip;
4532         uint64_t *regarray, idx, src;
4533
4534         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4535
4536         /* Structs. */
4537         regarray = instr_regarray_regarray(p, ip);
4538         idx = instr_regarray_idx_hbo(p, t, ip);
4539         src = instr_regarray_src_nbo(t, ip);
4540         regarray[idx] = src;
4541
4542         /* Thread. */
4543         thread_ip_inc(p);
4544 }
4545
4546 static inline void
4547 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4548 {
4549         struct thread *t = &p->threads[p->thread_id];
4550         struct instruction *ip = t->ip;
4551         uint64_t *regarray, idx, src;
4552
4553         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4554
4555         /* Structs. */
4556         regarray = instr_regarray_regarray(p, ip);
4557         idx = instr_regarray_idx_hbo(p, t, ip);
4558         src = instr_regarray_src_hbo(t, ip);
4559         regarray[idx] = src;
4560
4561         /* Thread. */
4562         thread_ip_inc(p);
4563 }
4564
4565 static inline void
4566 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4567 {
4568         struct thread *t = &p->threads[p->thread_id];
4569         struct instruction *ip = t->ip;
4570         uint64_t *regarray, idx, src;
4571
4572         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4573
4574         /* Structs. */
4575         regarray = instr_regarray_regarray(p, ip);
4576         idx = instr_regarray_idx_nbo(p, t, ip);
4577         src = ip->regarray.dstsrc_val;
4578         regarray[idx] = src;
4579
4580         /* Thread. */
4581         thread_ip_inc(p);
4582 }
4583
4584 static inline void
4585 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4586 {
4587         struct thread *t = &p->threads[p->thread_id];
4588         struct instruction *ip = t->ip;
4589         uint64_t *regarray, idx, src;
4590
4591         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4592
4593         /* Structs. */
4594         regarray = instr_regarray_regarray(p, ip);
4595         idx = instr_regarray_idx_hbo(p, t, ip);
4596         src = ip->regarray.dstsrc_val;
4597         regarray[idx] = src;
4598
4599         /* Thread. */
4600         thread_ip_inc(p);
4601 }
4602
4603 static inline void
4604 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4605 {
4606         struct thread *t = &p->threads[p->thread_id];
4607         struct instruction *ip = t->ip;
4608         uint64_t *regarray, idx, src;
4609
4610         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4611
4612         /* Structs. */
4613         regarray = instr_regarray_regarray(p, ip);
4614         idx = instr_regarray_idx_imm(p, ip);
4615         src = instr_regarray_src_nbo(t, ip);
4616         regarray[idx] = src;
4617
4618         /* Thread. */
4619         thread_ip_inc(p);
4620 }
4621
4622 static inline void
4623 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4624 {
4625         struct thread *t = &p->threads[p->thread_id];
4626         struct instruction *ip = t->ip;
4627         uint64_t *regarray, idx, src;
4628
4629         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4630
4631         /* Structs. */
4632         regarray = instr_regarray_regarray(p, ip);
4633         idx = instr_regarray_idx_imm(p, ip);
4634         src = instr_regarray_src_hbo(t, ip);
4635         regarray[idx] = src;
4636
4637         /* Thread. */
4638         thread_ip_inc(p);
4639 }
4640
4641 static inline void
4642 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4643 {
4644         struct thread *t = &p->threads[p->thread_id];
4645         struct instruction *ip = t->ip;
4646         uint64_t *regarray, idx, src;
4647
4648         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4649
4650         /* Structs. */
4651         regarray = instr_regarray_regarray(p, ip);
4652         idx = instr_regarray_idx_imm(p, ip);
4653         src = ip->regarray.dstsrc_val;
4654         regarray[idx] = src;
4655
4656         /* Thread. */
4657         thread_ip_inc(p);
4658 }
4659
4660 static inline void
4661 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4662 {
4663         struct thread *t = &p->threads[p->thread_id];
4664         struct instruction *ip = t->ip;
4665         uint64_t *regarray, idx, src;
4666
4667         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4668
4669         /* Structs. */
4670         regarray = instr_regarray_regarray(p, ip);
4671         idx = instr_regarray_idx_nbo(p, t, ip);
4672         src = instr_regarray_src_nbo(t, ip);
4673         regarray[idx] += src;
4674
4675         /* Thread. */
4676         thread_ip_inc(p);
4677 }
4678
4679 static inline void
4680 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4681 {
4682         struct thread *t = &p->threads[p->thread_id];
4683         struct instruction *ip = t->ip;
4684         uint64_t *regarray, idx, src;
4685
4686         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4687
4688         /* Structs. */
4689         regarray = instr_regarray_regarray(p, ip);
4690         idx = instr_regarray_idx_nbo(p, t, ip);
4691         src = instr_regarray_src_hbo(t, ip);
4692         regarray[idx] += src;
4693
4694         /* Thread. */
4695         thread_ip_inc(p);
4696 }
4697
4698 static inline void
4699 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4700 {
4701         struct thread *t = &p->threads[p->thread_id];
4702         struct instruction *ip = t->ip;
4703         uint64_t *regarray, idx, src;
4704
4705         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4706
4707         /* Structs. */
4708         regarray = instr_regarray_regarray(p, ip);
4709         idx = instr_regarray_idx_hbo(p, t, ip);
4710         src = instr_regarray_src_nbo(t, ip);
4711         regarray[idx] += src;
4712
4713         /* Thread. */
4714         thread_ip_inc(p);
4715 }
4716
4717 static inline void
4718 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4719 {
4720         struct thread *t = &p->threads[p->thread_id];
4721         struct instruction *ip = t->ip;
4722         uint64_t *regarray, idx, src;
4723
4724         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4725
4726         /* Structs. */
4727         regarray = instr_regarray_regarray(p, ip);
4728         idx = instr_regarray_idx_hbo(p, t, ip);
4729         src = instr_regarray_src_hbo(t, ip);
4730         regarray[idx] += src;
4731
4732         /* Thread. */
4733         thread_ip_inc(p);
4734 }
4735
4736 static inline void
4737 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4738 {
4739         struct thread *t = &p->threads[p->thread_id];
4740         struct instruction *ip = t->ip;
4741         uint64_t *regarray, idx, src;
4742
4743         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4744
4745         /* Structs. */
4746         regarray = instr_regarray_regarray(p, ip);
4747         idx = instr_regarray_idx_nbo(p, t, ip);
4748         src = ip->regarray.dstsrc_val;
4749         regarray[idx] += src;
4750
4751         /* Thread. */
4752         thread_ip_inc(p);
4753 }
4754
4755 static inline void
4756 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4757 {
4758         struct thread *t = &p->threads[p->thread_id];
4759         struct instruction *ip = t->ip;
4760         uint64_t *regarray, idx, src;
4761
4762         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4763
4764         /* Structs. */
4765         regarray = instr_regarray_regarray(p, ip);
4766         idx = instr_regarray_idx_hbo(p, t, ip);
4767         src = ip->regarray.dstsrc_val;
4768         regarray[idx] += src;
4769
4770         /* Thread. */
4771         thread_ip_inc(p);
4772 }
4773
4774 static inline void
4775 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4776 {
4777         struct thread *t = &p->threads[p->thread_id];
4778         struct instruction *ip = t->ip;
4779         uint64_t *regarray, idx, src;
4780
4781         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4782
4783         /* Structs. */
4784         regarray = instr_regarray_regarray(p, ip);
4785         idx = instr_regarray_idx_imm(p, ip);
4786         src = instr_regarray_src_nbo(t, ip);
4787         regarray[idx] += src;
4788
4789         /* Thread. */
4790         thread_ip_inc(p);
4791 }
4792
4793 static inline void
4794 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4795 {
4796         struct thread *t = &p->threads[p->thread_id];
4797         struct instruction *ip = t->ip;
4798         uint64_t *regarray, idx, src;
4799
4800         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4801
4802         /* Structs. */
4803         regarray = instr_regarray_regarray(p, ip);
4804         idx = instr_regarray_idx_imm(p, ip);
4805         src = instr_regarray_src_hbo(t, ip);
4806         regarray[idx] += src;
4807
4808         /* Thread. */
4809         thread_ip_inc(p);
4810 }
4811
4812 static inline void
4813 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4814 {
4815         struct thread *t = &p->threads[p->thread_id];
4816         struct instruction *ip = t->ip;
4817         uint64_t *regarray, idx, src;
4818
4819         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4820
4821         /* Structs. */
4822         regarray = instr_regarray_regarray(p, ip);
4823         idx = instr_regarray_idx_imm(p, ip);
4824         src = ip->regarray.dstsrc_val;
4825         regarray[idx] += src;
4826
4827         /* Thread. */
4828         thread_ip_inc(p);
4829 }
4830
4831 /*
4832  * metarray.
4833  */
4834 static struct metarray *
4835 metarray_find(struct rte_swx_pipeline *p, const char *name);
4836
4837 static int
4838 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4839                             struct action *action,
4840                             char **tokens,
4841                             int n_tokens,
4842                             struct instruction *instr,
4843                             struct instruction_data *data __rte_unused)
4844 {
4845         char *metarray = tokens[1], *idx = tokens[2];
4846         struct metarray *m;
4847         struct field *fidx;
4848         uint32_t idx_struct_id, idx_val;
4849
4850         CHECK(n_tokens == 3, EINVAL);
4851
4852         m = metarray_find(p, metarray);
4853         CHECK(m, EINVAL);
4854
4855         /* METPREFETCH_H, METPREFETCH_M. */
4856         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4857         if (fidx) {
4858                 CHECK(!fidx->var_size, EINVAL);
4859
4860                 instr->type = INSTR_METPREFETCH_M;
4861                 if (idx[0] == 'h')
4862                         instr->type = INSTR_METPREFETCH_H;
4863
4864                 instr->meter.metarray_id = m->id;
4865                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4866                 instr->meter.idx.n_bits = fidx->n_bits;
4867                 instr->meter.idx.offset = fidx->offset / 8;
4868                 return 0;
4869         }
4870
4871         /* METPREFETCH_I. */
4872         idx_val = strtoul(idx, &idx, 0);
4873         CHECK(!idx[0], EINVAL);
4874
4875         instr->type = INSTR_METPREFETCH_I;
4876         instr->meter.metarray_id = m->id;
4877         instr->meter.idx_val = idx_val;
4878         return 0;
4879 }
4880
4881 static int
4882 instr_meter_translate(struct rte_swx_pipeline *p,
4883                       struct action *action,
4884                       char **tokens,
4885                       int n_tokens,
4886                       struct instruction *instr,
4887                       struct instruction_data *data __rte_unused)
4888 {
4889         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4890         char *color_in = tokens[4], *color_out = tokens[5];
4891         struct metarray *m;
4892         struct field *fidx, *flength, *fcin, *fcout;
4893         uint32_t idx_struct_id, length_struct_id;
4894         uint32_t color_in_struct_id, color_out_struct_id;
4895
4896         CHECK(n_tokens == 6, EINVAL);
4897
4898         m = metarray_find(p, metarray);
4899         CHECK(m, EINVAL);
4900
4901         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4902
4903         flength = struct_field_parse(p, action, length, &length_struct_id);
4904         CHECK(flength, EINVAL);
4905         CHECK(!flength->var_size, EINVAL);
4906
4907         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4908
4909         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4910         CHECK(fcout, EINVAL);
4911         CHECK(!fcout->var_size, EINVAL);
4912
4913         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4914         if (fidx && fcin) {
4915                 CHECK(!fidx->var_size, EINVAL);
4916                 CHECK(!fcin->var_size, EINVAL);
4917
4918                 instr->type = INSTR_METER_MMM;
4919                 if (idx[0] == 'h' && length[0] == 'h')
4920                         instr->type = INSTR_METER_HHM;
4921                 if (idx[0] == 'h' && length[0] != 'h')
4922                         instr->type = INSTR_METER_HMM;
4923                 if (idx[0] != 'h' && length[0] == 'h')
4924                         instr->type = INSTR_METER_MHM;
4925
4926                 instr->meter.metarray_id = m->id;
4927
4928                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4929                 instr->meter.idx.n_bits = fidx->n_bits;
4930                 instr->meter.idx.offset = fidx->offset / 8;
4931
4932                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4933                 instr->meter.length.n_bits = flength->n_bits;
4934                 instr->meter.length.offset = flength->offset / 8;
4935
4936                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4937                 instr->meter.color_in.n_bits = fcin->n_bits;
4938                 instr->meter.color_in.offset = fcin->offset / 8;
4939
4940                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4941                 instr->meter.color_out.n_bits = fcout->n_bits;
4942                 instr->meter.color_out.offset = fcout->offset / 8;
4943
4944                 return 0;
4945         }
4946
4947         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4948         if (fidx && !fcin) {
4949                 uint32_t color_in_val;
4950
4951                 CHECK(!fidx->var_size, EINVAL);
4952
4953                 color_in_val = strtoul(color_in, &color_in, 0);
4954                 CHECK(!color_in[0], EINVAL);
4955
4956                 instr->type = INSTR_METER_MMI;
4957                 if (idx[0] == 'h' && length[0] == 'h')
4958                         instr->type = INSTR_METER_HHI;
4959                 if (idx[0] == 'h' && length[0] != 'h')
4960                         instr->type = INSTR_METER_HMI;
4961                 if (idx[0] != 'h' && length[0] == 'h')
4962                         instr->type = INSTR_METER_MHI;
4963
4964                 instr->meter.metarray_id = m->id;
4965
4966                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4967                 instr->meter.idx.n_bits = fidx->n_bits;
4968                 instr->meter.idx.offset = fidx->offset / 8;
4969
4970                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4971                 instr->meter.length.n_bits = flength->n_bits;
4972                 instr->meter.length.offset = flength->offset / 8;
4973
4974                 instr->meter.color_in_val = color_in_val;
4975
4976                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4977                 instr->meter.color_out.n_bits = fcout->n_bits;
4978                 instr->meter.color_out.offset = fcout->offset / 8;
4979
4980                 return 0;
4981         }
4982
4983         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4984         if (!fidx && fcin) {
4985                 uint32_t idx_val;
4986
4987                 idx_val = strtoul(idx, &idx, 0);
4988                 CHECK(!idx[0], EINVAL);
4989
4990                 CHECK(!fcin->var_size, EINVAL);
4991
4992                 instr->type = INSTR_METER_IMM;
4993                 if (length[0] == 'h')
4994                         instr->type = INSTR_METER_IHM;
4995
4996                 instr->meter.metarray_id = m->id;
4997
4998                 instr->meter.idx_val = idx_val;
4999
5000                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5001                 instr->meter.length.n_bits = flength->n_bits;
5002                 instr->meter.length.offset = flength->offset / 8;
5003
5004                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5005                 instr->meter.color_in.n_bits = fcin->n_bits;
5006                 instr->meter.color_in.offset = fcin->offset / 8;
5007
5008                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5009                 instr->meter.color_out.n_bits = fcout->n_bits;
5010                 instr->meter.color_out.offset = fcout->offset / 8;
5011
5012                 return 0;
5013         }
5014
5015         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5016         if (!fidx && !fcin) {
5017                 uint32_t idx_val, color_in_val;
5018
5019                 idx_val = strtoul(idx, &idx, 0);
5020                 CHECK(!idx[0], EINVAL);
5021
5022                 color_in_val = strtoul(color_in, &color_in, 0);
5023                 CHECK(!color_in[0], EINVAL);
5024
5025                 instr->type = INSTR_METER_IMI;
5026                 if (length[0] == 'h')
5027                         instr->type = INSTR_METER_IHI;
5028
5029                 instr->meter.metarray_id = m->id;
5030
5031                 instr->meter.idx_val = idx_val;
5032
5033                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5034                 instr->meter.length.n_bits = flength->n_bits;
5035                 instr->meter.length.offset = flength->offset / 8;
5036
5037                 instr->meter.color_in_val = color_in_val;
5038
5039                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5040                 instr->meter.color_out.n_bits = fcout->n_bits;
5041                 instr->meter.color_out.offset = fcout->offset / 8;
5042
5043                 return 0;
5044         }
5045
5046         CHECK(0, EINVAL);
5047 }
5048
5049 static inline struct meter *
5050 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5051 {
5052         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5053
5054         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5055         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5056         uint64_t idx64 = *idx64_ptr;
5057         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5058         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5059
5060         return &r->metarray[idx];
5061 }
5062
5063 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5064
5065 static inline struct meter *
5066 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5067 {
5068         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5069
5070         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5071         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5072         uint64_t idx64 = *idx64_ptr;
5073         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5074
5075         return &r->metarray[idx];
5076 }
5077
5078 #else
5079
5080 #define instr_meter_idx_nbo instr_meter_idx_hbo
5081
5082 #endif
5083
5084 static inline struct meter *
5085 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5086 {
5087         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5088
5089         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5090
5091         return &r->metarray[idx];
5092 }
5093
5094 static inline uint32_t
5095 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5096 {
5097         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5098         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5099         uint64_t src64 = *src64_ptr;
5100         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5101         uint64_t src = src64 & src64_mask;
5102
5103         return (uint32_t)src;
5104 }
5105
5106 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5107
5108 static inline uint32_t
5109 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5110 {
5111         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5112         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5113         uint64_t src64 = *src64_ptr;
5114         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5115
5116         return (uint32_t)src;
5117 }
5118
5119 #else
5120
5121 #define instr_meter_length_nbo instr_meter_length_hbo
5122
5123 #endif
5124
5125 static inline enum rte_color
5126 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5127 {
5128         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5129         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5130         uint64_t src64 = *src64_ptr;
5131         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5132         uint64_t src = src64 & src64_mask;
5133
5134         return (enum rte_color)src;
5135 }
5136
5137 static inline void
5138 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5139 {
5140         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5141         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5142         uint64_t dst64 = *dst64_ptr;
5143         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5144
5145         uint64_t src = (uint64_t)color_out;
5146
5147         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5148 }
5149
5150 static inline void
5151 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5152 {
5153         struct thread *t = &p->threads[p->thread_id];
5154         struct instruction *ip = t->ip;
5155         struct meter *m;
5156
5157         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5158
5159         /* Structs. */
5160         m = instr_meter_idx_nbo(p, t, ip);
5161         rte_prefetch0(m);
5162
5163         /* Thread. */
5164         thread_ip_inc(p);
5165 }
5166
5167 static inline void
5168 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5169 {
5170         struct thread *t = &p->threads[p->thread_id];
5171         struct instruction *ip = t->ip;
5172         struct meter *m;
5173
5174         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5175
5176         /* Structs. */
5177         m = instr_meter_idx_hbo(p, t, ip);
5178         rte_prefetch0(m);
5179
5180         /* Thread. */
5181         thread_ip_inc(p);
5182 }
5183
5184 static inline void
5185 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5186 {
5187         struct thread *t = &p->threads[p->thread_id];
5188         struct instruction *ip = t->ip;
5189         struct meter *m;
5190
5191         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5192
5193         /* Structs. */
5194         m = instr_meter_idx_imm(p, ip);
5195         rte_prefetch0(m);
5196
5197         /* Thread. */
5198         thread_ip_inc(p);
5199 }
5200
5201 static inline void
5202 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5203 {
5204         struct thread *t = &p->threads[p->thread_id];
5205         struct instruction *ip = t->ip;
5206         struct meter *m;
5207         uint64_t time, n_pkts, n_bytes;
5208         uint32_t length;
5209         enum rte_color color_in, color_out;
5210
5211         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5212
5213         /* Structs. */
5214         m = instr_meter_idx_nbo(p, t, ip);
5215         rte_prefetch0(m->n_pkts);
5216         time = rte_get_tsc_cycles();
5217         length = instr_meter_length_nbo(t, ip);
5218         color_in = instr_meter_color_in_hbo(t, ip);
5219
5220         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5221                 &m->profile->profile,
5222                 time,
5223                 length,
5224                 color_in);
5225
5226         color_out &= m->color_mask;
5227
5228         n_pkts = m->n_pkts[color_out];
5229         n_bytes = m->n_bytes[color_out];
5230
5231         instr_meter_color_out_hbo_set(t, ip, color_out);
5232
5233         m->n_pkts[color_out] = n_pkts + 1;
5234         m->n_bytes[color_out] = n_bytes + length;
5235
5236         /* Thread. */
5237         thread_ip_inc(p);
5238 }
5239
5240 static inline void
5241 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5242 {
5243         struct thread *t = &p->threads[p->thread_id];
5244         struct instruction *ip = t->ip;
5245         struct meter *m;
5246         uint64_t time, n_pkts, n_bytes;
5247         uint32_t length;
5248         enum rte_color color_in, color_out;
5249
5250         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5251
5252         /* Structs. */
5253         m = instr_meter_idx_nbo(p, t, ip);
5254         rte_prefetch0(m->n_pkts);
5255         time = rte_get_tsc_cycles();
5256         length = instr_meter_length_nbo(t, ip);
5257         color_in = (enum rte_color)ip->meter.color_in_val;
5258
5259         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5260                 &m->profile->profile,
5261                 time,
5262                 length,
5263                 color_in);
5264
5265         color_out &= m->color_mask;
5266
5267         n_pkts = m->n_pkts[color_out];
5268         n_bytes = m->n_bytes[color_out];
5269
5270         instr_meter_color_out_hbo_set(t, ip, color_out);
5271
5272         m->n_pkts[color_out] = n_pkts + 1;
5273         m->n_bytes[color_out] = n_bytes + length;
5274
5275         /* Thread. */
5276         thread_ip_inc(p);
5277 }
5278
5279 static inline void
5280 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5281 {
5282         struct thread *t = &p->threads[p->thread_id];
5283         struct instruction *ip = t->ip;
5284         struct meter *m;
5285         uint64_t time, n_pkts, n_bytes;
5286         uint32_t length;
5287         enum rte_color color_in, color_out;
5288
5289         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5290
5291         /* Structs. */
5292         m = instr_meter_idx_nbo(p, t, ip);
5293         rte_prefetch0(m->n_pkts);
5294         time = rte_get_tsc_cycles();
5295         length = instr_meter_length_hbo(t, ip);
5296         color_in = instr_meter_color_in_hbo(t, ip);
5297
5298         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5299                 &m->profile->profile,
5300                 time,
5301                 length,
5302                 color_in);
5303
5304         color_out &= m->color_mask;
5305
5306         n_pkts = m->n_pkts[color_out];
5307         n_bytes = m->n_bytes[color_out];
5308
5309         instr_meter_color_out_hbo_set(t, ip, color_out);
5310
5311         m->n_pkts[color_out] = n_pkts + 1;
5312         m->n_bytes[color_out] = n_bytes + length;
5313
5314         /* Thread. */
5315         thread_ip_inc(p);
5316 }
5317 static inline void
5318 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5319 {
5320         struct thread *t = &p->threads[p->thread_id];
5321         struct instruction *ip = t->ip;
5322         struct meter *m;
5323         uint64_t time, n_pkts, n_bytes;
5324         uint32_t length;
5325         enum rte_color color_in, color_out;
5326
5327         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5328
5329         /* Structs. */
5330         m = instr_meter_idx_nbo(p, t, ip);
5331         rte_prefetch0(m->n_pkts);
5332         time = rte_get_tsc_cycles();
5333         length = instr_meter_length_hbo(t, ip);
5334         color_in = (enum rte_color)ip->meter.color_in_val;
5335
5336         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5337                 &m->profile->profile,
5338                 time,
5339                 length,
5340                 color_in);
5341
5342         color_out &= m->color_mask;
5343
5344         n_pkts = m->n_pkts[color_out];
5345         n_bytes = m->n_bytes[color_out];
5346
5347         instr_meter_color_out_hbo_set(t, ip, color_out);
5348
5349         m->n_pkts[color_out] = n_pkts + 1;
5350         m->n_bytes[color_out] = n_bytes + length;
5351
5352         /* Thread. */
5353         thread_ip_inc(p);
5354 }
5355
5356 static inline void
5357 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5358 {
5359         struct thread *t = &p->threads[p->thread_id];
5360         struct instruction *ip = t->ip;
5361         struct meter *m;
5362         uint64_t time, n_pkts, n_bytes;
5363         uint32_t length;
5364         enum rte_color color_in, color_out;
5365
5366         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5367
5368         /* Structs. */
5369         m = instr_meter_idx_hbo(p, t, ip);
5370         rte_prefetch0(m->n_pkts);
5371         time = rte_get_tsc_cycles();
5372         length = instr_meter_length_nbo(t, ip);
5373         color_in = instr_meter_color_in_hbo(t, ip);
5374
5375         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5376                 &m->profile->profile,
5377                 time,
5378                 length,
5379                 color_in);
5380
5381         color_out &= m->color_mask;
5382
5383         n_pkts = m->n_pkts[color_out];
5384         n_bytes = m->n_bytes[color_out];
5385
5386         instr_meter_color_out_hbo_set(t, ip, color_out);
5387
5388         m->n_pkts[color_out] = n_pkts + 1;
5389         m->n_bytes[color_out] = n_bytes + length;
5390
5391         /* Thread. */
5392         thread_ip_inc(p);
5393 }
5394
5395 static inline void
5396 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5397 {
5398         struct thread *t = &p->threads[p->thread_id];
5399         struct instruction *ip = t->ip;
5400         struct meter *m;
5401         uint64_t time, n_pkts, n_bytes;
5402         uint32_t length;
5403         enum rte_color color_in, color_out;
5404
5405         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5406
5407         /* Structs. */
5408         m = instr_meter_idx_hbo(p, t, ip);
5409         rte_prefetch0(m->n_pkts);
5410         time = rte_get_tsc_cycles();
5411         length = instr_meter_length_nbo(t, ip);
5412         color_in = (enum rte_color)ip->meter.color_in_val;
5413
5414         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5415                 &m->profile->profile,
5416                 time,
5417                 length,
5418                 color_in);
5419
5420         color_out &= m->color_mask;
5421
5422         n_pkts = m->n_pkts[color_out];
5423         n_bytes = m->n_bytes[color_out];
5424
5425         instr_meter_color_out_hbo_set(t, ip, color_out);
5426
5427         m->n_pkts[color_out] = n_pkts + 1;
5428         m->n_bytes[color_out] = n_bytes + length;
5429
5430         /* Thread. */
5431         thread_ip_inc(p);
5432 }
5433
5434 static inline void
5435 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5436 {
5437         struct thread *t = &p->threads[p->thread_id];
5438         struct instruction *ip = t->ip;
5439         struct meter *m;
5440         uint64_t time, n_pkts, n_bytes;
5441         uint32_t length;
5442         enum rte_color color_in, color_out;
5443
5444         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5445
5446         /* Structs. */
5447         m = instr_meter_idx_hbo(p, t, ip);
5448         rte_prefetch0(m->n_pkts);
5449         time = rte_get_tsc_cycles();
5450         length = instr_meter_length_hbo(t, ip);
5451         color_in = instr_meter_color_in_hbo(t, ip);
5452
5453         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5454                 &m->profile->profile,
5455                 time,
5456                 length,
5457                 color_in);
5458
5459         color_out &= m->color_mask;
5460
5461         n_pkts = m->n_pkts[color_out];
5462         n_bytes = m->n_bytes[color_out];
5463
5464         instr_meter_color_out_hbo_set(t, ip, color_out);
5465
5466         m->n_pkts[color_out] = n_pkts + 1;
5467         m->n_bytes[color_out] = n_bytes + length;
5468
5469         /* Thread. */
5470         thread_ip_inc(p);
5471 }
5472
5473 static inline void
5474 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5475 {
5476         struct thread *t = &p->threads[p->thread_id];
5477         struct instruction *ip = t->ip;
5478         struct meter *m;
5479         uint64_t time, n_pkts, n_bytes;
5480         uint32_t length;
5481         enum rte_color color_in, color_out;
5482
5483         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5484
5485         /* Structs. */
5486         m = instr_meter_idx_hbo(p, t, ip);
5487         rte_prefetch0(m->n_pkts);
5488         time = rte_get_tsc_cycles();
5489         length = instr_meter_length_hbo(t, ip);
5490         color_in = (enum rte_color)ip->meter.color_in_val;
5491
5492         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5493                 &m->profile->profile,
5494                 time,
5495                 length,
5496                 color_in);
5497
5498         color_out &= m->color_mask;
5499
5500         n_pkts = m->n_pkts[color_out];
5501         n_bytes = m->n_bytes[color_out];
5502
5503         instr_meter_color_out_hbo_set(t, ip, color_out);
5504
5505         m->n_pkts[color_out] = n_pkts + 1;
5506         m->n_bytes[color_out] = n_bytes + length;
5507
5508         /* Thread. */
5509         thread_ip_inc(p);
5510 }
5511
5512 static inline void
5513 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5514 {
5515         struct thread *t = &p->threads[p->thread_id];
5516         struct instruction *ip = t->ip;
5517         struct meter *m;
5518         uint64_t time, n_pkts, n_bytes;
5519         uint32_t length;
5520         enum rte_color color_in, color_out;
5521
5522         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5523
5524         /* Structs. */
5525         m = instr_meter_idx_imm(p, ip);
5526         rte_prefetch0(m->n_pkts);
5527         time = rte_get_tsc_cycles();
5528         length = instr_meter_length_nbo(t, ip);
5529         color_in = instr_meter_color_in_hbo(t, ip);
5530
5531         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5532                 &m->profile->profile,
5533                 time,
5534                 length,
5535                 color_in);
5536
5537         color_out &= m->color_mask;
5538
5539         n_pkts = m->n_pkts[color_out];
5540         n_bytes = m->n_bytes[color_out];
5541
5542         instr_meter_color_out_hbo_set(t, ip, color_out);
5543
5544         m->n_pkts[color_out] = n_pkts + 1;
5545         m->n_bytes[color_out] = n_bytes + length;
5546
5547         /* Thread. */
5548         thread_ip_inc(p);
5549 }
5550
5551 static inline void
5552 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5553 {
5554         struct thread *t = &p->threads[p->thread_id];
5555         struct instruction *ip = t->ip;
5556         struct meter *m;
5557         uint64_t time, n_pkts, n_bytes;
5558         uint32_t length;
5559         enum rte_color color_in, color_out;
5560
5561         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5562
5563         /* Structs. */
5564         m = instr_meter_idx_imm(p, ip);
5565         rte_prefetch0(m->n_pkts);
5566         time = rte_get_tsc_cycles();
5567         length = instr_meter_length_nbo(t, ip);
5568         color_in = (enum rte_color)ip->meter.color_in_val;
5569
5570         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5571                 &m->profile->profile,
5572                 time,
5573                 length,
5574                 color_in);
5575
5576         color_out &= m->color_mask;
5577
5578         n_pkts = m->n_pkts[color_out];
5579         n_bytes = m->n_bytes[color_out];
5580
5581         instr_meter_color_out_hbo_set(t, ip, color_out);
5582
5583         m->n_pkts[color_out] = n_pkts + 1;
5584         m->n_bytes[color_out] = n_bytes + length;
5585
5586         /* Thread. */
5587         thread_ip_inc(p);
5588 }
5589
5590 static inline void
5591 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5592 {
5593         struct thread *t = &p->threads[p->thread_id];
5594         struct instruction *ip = t->ip;
5595         struct meter *m;
5596         uint64_t time, n_pkts, n_bytes;
5597         uint32_t length;
5598         enum rte_color color_in, color_out;
5599
5600         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5601
5602         /* Structs. */
5603         m = instr_meter_idx_imm(p, ip);
5604         rte_prefetch0(m->n_pkts);
5605         time = rte_get_tsc_cycles();
5606         length = instr_meter_length_hbo(t, ip);
5607         color_in = instr_meter_color_in_hbo(t, ip);
5608
5609         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5610                 &m->profile->profile,
5611                 time,
5612                 length,
5613                 color_in);
5614
5615         color_out &= m->color_mask;
5616
5617         n_pkts = m->n_pkts[color_out];
5618         n_bytes = m->n_bytes[color_out];
5619
5620         instr_meter_color_out_hbo_set(t, ip, color_out);
5621
5622         m->n_pkts[color_out] = n_pkts + 1;
5623         m->n_bytes[color_out] = n_bytes + length;
5624
5625         /* Thread. */
5626         thread_ip_inc(p);
5627 }
5628 static inline void
5629 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5630 {
5631         struct thread *t = &p->threads[p->thread_id];
5632         struct instruction *ip = t->ip;
5633         struct meter *m;
5634         uint64_t time, n_pkts, n_bytes;
5635         uint32_t length;
5636         enum rte_color color_in, color_out;
5637
5638         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5639
5640         /* Structs. */
5641         m = instr_meter_idx_imm(p, ip);
5642         rte_prefetch0(m->n_pkts);
5643         time = rte_get_tsc_cycles();
5644         length = instr_meter_length_hbo(t, ip);
5645         color_in = (enum rte_color)ip->meter.color_in_val;
5646
5647         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5648                 &m->profile->profile,
5649                 time,
5650                 length,
5651                 color_in);
5652
5653         color_out &= m->color_mask;
5654
5655         n_pkts = m->n_pkts[color_out];
5656         n_bytes = m->n_bytes[color_out];
5657
5658         instr_meter_color_out_hbo_set(t, ip, color_out);
5659
5660         m->n_pkts[color_out] = n_pkts + 1;
5661         m->n_bytes[color_out] = n_bytes + length;
5662
5663         /* Thread. */
5664         thread_ip_inc(p);
5665 }
5666
5667 /*
5668  * jmp.
5669  */
5670 static int
5671 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5672                     struct action *action __rte_unused,
5673                     char **tokens,
5674                     int n_tokens,
5675                     struct instruction *instr,
5676                     struct instruction_data *data)
5677 {
5678         CHECK(n_tokens == 2, EINVAL);
5679
5680         strcpy(data->jmp_label, tokens[1]);
5681
5682         instr->type = INSTR_JMP;
5683         instr->jmp.ip = NULL; /* Resolved later. */
5684         return 0;
5685 }
5686
5687 static int
5688 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5689                           struct action *action __rte_unused,
5690                           char **tokens,
5691                           int n_tokens,
5692                           struct instruction *instr,
5693                           struct instruction_data *data)
5694 {
5695         struct header *h;
5696
5697         CHECK(n_tokens == 3, EINVAL);
5698
5699         strcpy(data->jmp_label, tokens[1]);
5700
5701         h = header_parse(p, tokens[2]);
5702         CHECK(h, EINVAL);
5703
5704         instr->type = INSTR_JMP_VALID;
5705         instr->jmp.ip = NULL; /* Resolved later. */
5706         instr->jmp.header_id = h->id;
5707         return 0;
5708 }
5709
5710 static int
5711 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5712                             struct action *action __rte_unused,
5713                             char **tokens,
5714                             int n_tokens,
5715                             struct instruction *instr,
5716                             struct instruction_data *data)
5717 {
5718         struct header *h;
5719
5720         CHECK(n_tokens == 3, EINVAL);
5721
5722         strcpy(data->jmp_label, tokens[1]);
5723
5724         h = header_parse(p, tokens[2]);
5725         CHECK(h, EINVAL);
5726
5727         instr->type = INSTR_JMP_INVALID;
5728         instr->jmp.ip = NULL; /* Resolved later. */
5729         instr->jmp.header_id = h->id;
5730         return 0;
5731 }
5732
5733 static int
5734 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5735                         struct action *action,
5736                         char **tokens,
5737                         int n_tokens,
5738                         struct instruction *instr,
5739                         struct instruction_data *data)
5740 {
5741         CHECK(!action, EINVAL);
5742         CHECK(n_tokens == 2, EINVAL);
5743
5744         strcpy(data->jmp_label, tokens[1]);
5745
5746         instr->type = INSTR_JMP_HIT;
5747         instr->jmp.ip = NULL; /* Resolved later. */
5748         return 0;
5749 }
5750
5751 static int
5752 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5753                          struct action *action,
5754                          char **tokens,
5755                          int n_tokens,
5756                          struct instruction *instr,
5757                          struct instruction_data *data)
5758 {
5759         CHECK(!action, EINVAL);
5760         CHECK(n_tokens == 2, EINVAL);
5761
5762         strcpy(data->jmp_label, tokens[1]);
5763
5764         instr->type = INSTR_JMP_MISS;
5765         instr->jmp.ip = NULL; /* Resolved later. */
5766         return 0;
5767 }
5768
5769 static int
5770 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5771                                struct action *action,
5772                                char **tokens,
5773                                int n_tokens,
5774                                struct instruction *instr,
5775                                struct instruction_data *data)
5776 {
5777         struct action *a;
5778
5779         CHECK(!action, EINVAL);
5780         CHECK(n_tokens == 3, EINVAL);
5781
5782         strcpy(data->jmp_label, tokens[1]);
5783
5784         a = action_find(p, tokens[2]);
5785         CHECK(a, EINVAL);
5786
5787         instr->type = INSTR_JMP_ACTION_HIT;
5788         instr->jmp.ip = NULL; /* Resolved later. */
5789         instr->jmp.action_id = a->id;
5790         return 0;
5791 }
5792
5793 static int
5794 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5795                                 struct action *action,
5796                                 char **tokens,
5797                                 int n_tokens,
5798                                 struct instruction *instr,
5799                                 struct instruction_data *data)
5800 {
5801         struct action *a;
5802
5803         CHECK(!action, EINVAL);
5804         CHECK(n_tokens == 3, EINVAL);
5805
5806         strcpy(data->jmp_label, tokens[1]);
5807
5808         a = action_find(p, tokens[2]);
5809         CHECK(a, EINVAL);
5810
5811         instr->type = INSTR_JMP_ACTION_MISS;
5812         instr->jmp.ip = NULL; /* Resolved later. */
5813         instr->jmp.action_id = a->id;
5814         return 0;
5815 }
5816
5817 static int
5818 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5819                        struct action *action,
5820                        char **tokens,
5821                        int n_tokens,
5822                        struct instruction *instr,
5823                        struct instruction_data *data)
5824 {
5825         char *a = tokens[2], *b = tokens[3];
5826         struct field *fa, *fb;
5827         uint64_t b_val;
5828         uint32_t a_struct_id, b_struct_id;
5829
5830         CHECK(n_tokens == 4, EINVAL);
5831
5832         strcpy(data->jmp_label, tokens[1]);
5833
5834         fa = struct_field_parse(p, action, a, &a_struct_id);
5835         CHECK(fa, EINVAL);
5836         CHECK(!fa->var_size, EINVAL);
5837
5838         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5839         fb = struct_field_parse(p, action, b, &b_struct_id);
5840         if (fb) {
5841                 CHECK(!fb->var_size, EINVAL);
5842
5843                 instr->type = INSTR_JMP_EQ;
5844                 if (a[0] != 'h' && b[0] == 'h')
5845                         instr->type = INSTR_JMP_EQ_MH;
5846                 if (a[0] == 'h' && b[0] != 'h')
5847                         instr->type = INSTR_JMP_EQ_HM;
5848                 if (a[0] == 'h' && b[0] == 'h')
5849                         instr->type = INSTR_JMP_EQ_HH;
5850                 instr->jmp.ip = NULL; /* Resolved later. */
5851
5852                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5853                 instr->jmp.a.n_bits = fa->n_bits;
5854                 instr->jmp.a.offset = fa->offset / 8;
5855                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5856                 instr->jmp.b.n_bits = fb->n_bits;
5857                 instr->jmp.b.offset = fb->offset / 8;
5858                 return 0;
5859         }
5860
5861         /* JMP_EQ_I. */
5862         b_val = strtoull(b, &b, 0);
5863         CHECK(!b[0], EINVAL);
5864
5865         if (a[0] == 'h')
5866                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5867
5868         instr->type = INSTR_JMP_EQ_I;
5869         instr->jmp.ip = NULL; /* Resolved later. */
5870         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5871         instr->jmp.a.n_bits = fa->n_bits;
5872         instr->jmp.a.offset = fa->offset / 8;
5873         instr->jmp.b_val = b_val;
5874         return 0;
5875 }
5876
5877 static int
5878 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5879                         struct action *action,
5880                         char **tokens,
5881                         int n_tokens,
5882                         struct instruction *instr,
5883                         struct instruction_data *data)
5884 {
5885         char *a = tokens[2], *b = tokens[3];
5886         struct field *fa, *fb;
5887         uint64_t b_val;
5888         uint32_t a_struct_id, b_struct_id;
5889
5890         CHECK(n_tokens == 4, EINVAL);
5891
5892         strcpy(data->jmp_label, tokens[1]);
5893
5894         fa = struct_field_parse(p, action, a, &a_struct_id);
5895         CHECK(fa, EINVAL);
5896         CHECK(!fa->var_size, EINVAL);
5897
5898         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5899         fb = struct_field_parse(p, action, b, &b_struct_id);
5900         if (fb) {
5901                 CHECK(!fb->var_size, EINVAL);
5902
5903                 instr->type = INSTR_JMP_NEQ;
5904                 if (a[0] != 'h' && b[0] == 'h')
5905                         instr->type = INSTR_JMP_NEQ_MH;
5906                 if (a[0] == 'h' && b[0] != 'h')
5907                         instr->type = INSTR_JMP_NEQ_HM;
5908                 if (a[0] == 'h' && b[0] == 'h')
5909                         instr->type = INSTR_JMP_NEQ_HH;
5910                 instr->jmp.ip = NULL; /* Resolved later. */
5911
5912                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5913                 instr->jmp.a.n_bits = fa->n_bits;
5914                 instr->jmp.a.offset = fa->offset / 8;
5915                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5916                 instr->jmp.b.n_bits = fb->n_bits;
5917                 instr->jmp.b.offset = fb->offset / 8;
5918                 return 0;
5919         }
5920
5921         /* JMP_NEQ_I. */
5922         b_val = strtoull(b, &b, 0);
5923         CHECK(!b[0], EINVAL);
5924
5925         if (a[0] == 'h')
5926                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5927
5928         instr->type = INSTR_JMP_NEQ_I;
5929         instr->jmp.ip = NULL; /* Resolved later. */
5930         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5931         instr->jmp.a.n_bits = fa->n_bits;
5932         instr->jmp.a.offset = fa->offset / 8;
5933         instr->jmp.b_val = b_val;
5934         return 0;
5935 }
5936
5937 static int
5938 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5939                        struct action *action,
5940                        char **tokens,
5941                        int n_tokens,
5942                        struct instruction *instr,
5943                        struct instruction_data *data)
5944 {
5945         char *a = tokens[2], *b = tokens[3];
5946         struct field *fa, *fb;
5947         uint64_t b_val;
5948         uint32_t a_struct_id, b_struct_id;
5949
5950         CHECK(n_tokens == 4, EINVAL);
5951
5952         strcpy(data->jmp_label, tokens[1]);
5953
5954         fa = struct_field_parse(p, action, a, &a_struct_id);
5955         CHECK(fa, EINVAL);
5956         CHECK(!fa->var_size, EINVAL);
5957
5958         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5959         fb = struct_field_parse(p, action, b, &b_struct_id);
5960         if (fb) {
5961                 CHECK(!fb->var_size, EINVAL);
5962
5963                 instr->type = INSTR_JMP_LT;
5964                 if (a[0] == 'h' && b[0] != 'h')
5965                         instr->type = INSTR_JMP_LT_HM;
5966                 if (a[0] != 'h' && b[0] == 'h')
5967                         instr->type = INSTR_JMP_LT_MH;
5968                 if (a[0] == 'h' && b[0] == 'h')
5969                         instr->type = INSTR_JMP_LT_HH;
5970                 instr->jmp.ip = NULL; /* Resolved later. */
5971
5972                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5973                 instr->jmp.a.n_bits = fa->n_bits;
5974                 instr->jmp.a.offset = fa->offset / 8;
5975                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5976                 instr->jmp.b.n_bits = fb->n_bits;
5977                 instr->jmp.b.offset = fb->offset / 8;
5978                 return 0;
5979         }
5980
5981         /* JMP_LT_MI, JMP_LT_HI. */
5982         b_val = strtoull(b, &b, 0);
5983         CHECK(!b[0], EINVAL);
5984
5985         instr->type = INSTR_JMP_LT_MI;
5986         if (a[0] == 'h')
5987                 instr->type = INSTR_JMP_LT_HI;
5988         instr->jmp.ip = NULL; /* Resolved later. */
5989
5990         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5991         instr->jmp.a.n_bits = fa->n_bits;
5992         instr->jmp.a.offset = fa->offset / 8;
5993         instr->jmp.b_val = b_val;
5994         return 0;
5995 }
5996
5997 static int
5998 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5999                        struct action *action,
6000                        char **tokens,
6001                        int n_tokens,
6002                        struct instruction *instr,
6003                        struct instruction_data *data)
6004 {
6005         char *a = tokens[2], *b = tokens[3];
6006         struct field *fa, *fb;
6007         uint64_t b_val;
6008         uint32_t a_struct_id, b_struct_id;
6009
6010         CHECK(n_tokens == 4, EINVAL);
6011
6012         strcpy(data->jmp_label, tokens[1]);
6013
6014         fa = struct_field_parse(p, action, a, &a_struct_id);
6015         CHECK(fa, EINVAL);
6016         CHECK(!fa->var_size, EINVAL);
6017
6018         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6019         fb = struct_field_parse(p, action, b, &b_struct_id);
6020         if (fb) {
6021                 CHECK(!fb->var_size, EINVAL);
6022
6023                 instr->type = INSTR_JMP_GT;
6024                 if (a[0] == 'h' && b[0] != 'h')
6025                         instr->type = INSTR_JMP_GT_HM;
6026                 if (a[0] != 'h' && b[0] == 'h')
6027                         instr->type = INSTR_JMP_GT_MH;
6028                 if (a[0] == 'h' && b[0] == 'h')
6029                         instr->type = INSTR_JMP_GT_HH;
6030                 instr->jmp.ip = NULL; /* Resolved later. */
6031
6032                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6033                 instr->jmp.a.n_bits = fa->n_bits;
6034                 instr->jmp.a.offset = fa->offset / 8;
6035                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6036                 instr->jmp.b.n_bits = fb->n_bits;
6037                 instr->jmp.b.offset = fb->offset / 8;
6038                 return 0;
6039         }
6040
6041         /* JMP_GT_MI, JMP_GT_HI. */
6042         b_val = strtoull(b, &b, 0);
6043         CHECK(!b[0], EINVAL);
6044
6045         instr->type = INSTR_JMP_GT_MI;
6046         if (a[0] == 'h')
6047                 instr->type = INSTR_JMP_GT_HI;
6048         instr->jmp.ip = NULL; /* Resolved later. */
6049
6050         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6051         instr->jmp.a.n_bits = fa->n_bits;
6052         instr->jmp.a.offset = fa->offset / 8;
6053         instr->jmp.b_val = b_val;
6054         return 0;
6055 }
6056
6057 static inline void
6058 instr_jmp_exec(struct rte_swx_pipeline *p)
6059 {
6060         struct thread *t = &p->threads[p->thread_id];
6061         struct instruction *ip = t->ip;
6062
6063         TRACE("[Thread %2u] jmp\n", p->thread_id);
6064
6065         thread_ip_set(t, ip->jmp.ip);
6066 }
6067
6068 static inline void
6069 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6070 {
6071         struct thread *t = &p->threads[p->thread_id];
6072         struct instruction *ip = t->ip;
6073         uint32_t header_id = ip->jmp.header_id;
6074
6075         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6076
6077         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6078 }
6079
6080 static inline void
6081 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6082 {
6083         struct thread *t = &p->threads[p->thread_id];
6084         struct instruction *ip = t->ip;
6085         uint32_t header_id = ip->jmp.header_id;
6086
6087         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6088
6089         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6090 }
6091
6092 static inline void
6093 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6094 {
6095         struct thread *t = &p->threads[p->thread_id];
6096         struct instruction *ip = t->ip;
6097         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6098
6099         TRACE("[Thread %2u] jmph\n", p->thread_id);
6100
6101         t->ip = ip_next[t->hit];
6102 }
6103
6104 static inline void
6105 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6106 {
6107         struct thread *t = &p->threads[p->thread_id];
6108         struct instruction *ip = t->ip;
6109         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6110
6111         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6112
6113         t->ip = ip_next[t->hit];
6114 }
6115
6116 static inline void
6117 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6118 {
6119         struct thread *t = &p->threads[p->thread_id];
6120         struct instruction *ip = t->ip;
6121
6122         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6123
6124         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6125 }
6126
6127 static inline void
6128 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6129 {
6130         struct thread *t = &p->threads[p->thread_id];
6131         struct instruction *ip = t->ip;
6132
6133         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6134
6135         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6136 }
6137
6138 static inline void
6139 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6140 {
6141         struct thread *t = &p->threads[p->thread_id];
6142         struct instruction *ip = t->ip;
6143
6144         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6145
6146         JMP_CMP(t, ip, ==);
6147 }
6148
6149 static inline void
6150 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6151 {
6152         struct thread *t = &p->threads[p->thread_id];
6153         struct instruction *ip = t->ip;
6154
6155         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6156
6157         JMP_CMP_MH(t, ip, ==);
6158 }
6159
6160 static inline void
6161 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6162 {
6163         struct thread *t = &p->threads[p->thread_id];
6164         struct instruction *ip = t->ip;
6165
6166         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6167
6168         JMP_CMP_HM(t, ip, ==);
6169 }
6170
6171 static inline void
6172 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6173 {
6174         struct thread *t = &p->threads[p->thread_id];
6175         struct instruction *ip = t->ip;
6176
6177         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6178
6179         JMP_CMP_HH_FAST(t, ip, ==);
6180 }
6181
6182 static inline void
6183 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6184 {
6185         struct thread *t = &p->threads[p->thread_id];
6186         struct instruction *ip = t->ip;
6187
6188         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6189
6190         JMP_CMP_I(t, ip, ==);
6191 }
6192
6193 static inline void
6194 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6195 {
6196         struct thread *t = &p->threads[p->thread_id];
6197         struct instruction *ip = t->ip;
6198
6199         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6200
6201         JMP_CMP(t, ip, !=);
6202 }
6203
6204 static inline void
6205 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6206 {
6207         struct thread *t = &p->threads[p->thread_id];
6208         struct instruction *ip = t->ip;
6209
6210         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6211
6212         JMP_CMP_MH(t, ip, !=);
6213 }
6214
6215 static inline void
6216 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6217 {
6218         struct thread *t = &p->threads[p->thread_id];
6219         struct instruction *ip = t->ip;
6220
6221         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6222
6223         JMP_CMP_HM(t, ip, !=);
6224 }
6225
6226 static inline void
6227 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6228 {
6229         struct thread *t = &p->threads[p->thread_id];
6230         struct instruction *ip = t->ip;
6231
6232         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6233
6234         JMP_CMP_HH_FAST(t, ip, !=);
6235 }
6236
6237 static inline void
6238 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6239 {
6240         struct thread *t = &p->threads[p->thread_id];
6241         struct instruction *ip = t->ip;
6242
6243         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6244
6245         JMP_CMP_I(t, ip, !=);
6246 }
6247
6248 static inline void
6249 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6250 {
6251         struct thread *t = &p->threads[p->thread_id];
6252         struct instruction *ip = t->ip;
6253
6254         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6255
6256         JMP_CMP(t, ip, <);
6257 }
6258
6259 static inline void
6260 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6261 {
6262         struct thread *t = &p->threads[p->thread_id];
6263         struct instruction *ip = t->ip;
6264
6265         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6266
6267         JMP_CMP_MH(t, ip, <);
6268 }
6269
6270 static inline void
6271 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6272 {
6273         struct thread *t = &p->threads[p->thread_id];
6274         struct instruction *ip = t->ip;
6275
6276         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6277
6278         JMP_CMP_HM(t, ip, <);
6279 }
6280
6281 static inline void
6282 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6283 {
6284         struct thread *t = &p->threads[p->thread_id];
6285         struct instruction *ip = t->ip;
6286
6287         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6288
6289         JMP_CMP_HH(t, ip, <);
6290 }
6291
6292 static inline void
6293 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6294 {
6295         struct thread *t = &p->threads[p->thread_id];
6296         struct instruction *ip = t->ip;
6297
6298         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6299
6300         JMP_CMP_MI(t, ip, <);
6301 }
6302
6303 static inline void
6304 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6305 {
6306         struct thread *t = &p->threads[p->thread_id];
6307         struct instruction *ip = t->ip;
6308
6309         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6310
6311         JMP_CMP_HI(t, ip, <);
6312 }
6313
6314 static inline void
6315 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6316 {
6317         struct thread *t = &p->threads[p->thread_id];
6318         struct instruction *ip = t->ip;
6319
6320         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6321
6322         JMP_CMP(t, ip, >);
6323 }
6324
6325 static inline void
6326 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6327 {
6328         struct thread *t = &p->threads[p->thread_id];
6329         struct instruction *ip = t->ip;
6330
6331         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6332
6333         JMP_CMP_MH(t, ip, >);
6334 }
6335
6336 static inline void
6337 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6338 {
6339         struct thread *t = &p->threads[p->thread_id];
6340         struct instruction *ip = t->ip;
6341
6342         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6343
6344         JMP_CMP_HM(t, ip, >);
6345 }
6346
6347 static inline void
6348 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6349 {
6350         struct thread *t = &p->threads[p->thread_id];
6351         struct instruction *ip = t->ip;
6352
6353         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6354
6355         JMP_CMP_HH(t, ip, >);
6356 }
6357
6358 static inline void
6359 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6360 {
6361         struct thread *t = &p->threads[p->thread_id];
6362         struct instruction *ip = t->ip;
6363
6364         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6365
6366         JMP_CMP_MI(t, ip, >);
6367 }
6368
6369 static inline void
6370 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6371 {
6372         struct thread *t = &p->threads[p->thread_id];
6373         struct instruction *ip = t->ip;
6374
6375         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6376
6377         JMP_CMP_HI(t, ip, >);
6378 }
6379
6380 /*
6381  * return.
6382  */
6383 static int
6384 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6385                        struct action *action,
6386                        char **tokens __rte_unused,
6387                        int n_tokens,
6388                        struct instruction *instr,
6389                        struct instruction_data *data __rte_unused)
6390 {
6391         CHECK(action, EINVAL);
6392         CHECK(n_tokens == 1, EINVAL);
6393
6394         instr->type = INSTR_RETURN;
6395         return 0;
6396 }
6397
6398 static inline void
6399 instr_return_exec(struct rte_swx_pipeline *p)
6400 {
6401         struct thread *t = &p->threads[p->thread_id];
6402
6403         TRACE("[Thread %2u] return\n", p->thread_id);
6404
6405         t->ip = t->ret;
6406 }
6407
6408 static int
6409 instr_translate(struct rte_swx_pipeline *p,
6410                 struct action *action,
6411                 char *string,
6412                 struct instruction *instr,
6413                 struct instruction_data *data)
6414 {
6415         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6416         int n_tokens = 0, tpos = 0;
6417
6418         /* Parse the instruction string into tokens. */
6419         for ( ; ; ) {
6420                 char *token;
6421
6422                 token = strtok_r(string, " \t\v", &string);
6423                 if (!token)
6424                         break;
6425
6426                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6427                 CHECK_NAME(token, EINVAL);
6428
6429                 tokens[n_tokens] = token;
6430                 n_tokens++;
6431         }
6432
6433         CHECK(n_tokens, EINVAL);
6434
6435         /* Handle the optional instruction label. */
6436         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6437                 strcpy(data->label, tokens[0]);
6438
6439                 tpos += 2;
6440                 CHECK(n_tokens - tpos, EINVAL);
6441         }
6442
6443         /* Identify the instruction type. */
6444         if (!strcmp(tokens[tpos], "rx"))
6445                 return instr_rx_translate(p,
6446                                           action,
6447                                           &tokens[tpos],
6448                                           n_tokens - tpos,
6449                                           instr,
6450                                           data);
6451
6452         if (!strcmp(tokens[tpos], "tx"))
6453                 return instr_tx_translate(p,
6454                                           action,
6455                                           &tokens[tpos],
6456                                           n_tokens - tpos,
6457                                           instr,
6458                                           data);
6459
6460         if (!strcmp(tokens[tpos], "drop"))
6461                 return instr_drop_translate(p,
6462                                             action,
6463                                             &tokens[tpos],
6464                                             n_tokens - tpos,
6465                                             instr,
6466                                             data);
6467
6468         if (!strcmp(tokens[tpos], "extract"))
6469                 return instr_hdr_extract_translate(p,
6470                                                    action,
6471                                                    &tokens[tpos],
6472                                                    n_tokens - tpos,
6473                                                    instr,
6474                                                    data);
6475
6476         if (!strcmp(tokens[tpos], "lookahead"))
6477                 return instr_hdr_lookahead_translate(p,
6478                                                      action,
6479                                                      &tokens[tpos],
6480                                                      n_tokens - tpos,
6481                                                      instr,
6482                                                      data);
6483
6484         if (!strcmp(tokens[tpos], "emit"))
6485                 return instr_hdr_emit_translate(p,
6486                                                 action,
6487                                                 &tokens[tpos],
6488                                                 n_tokens - tpos,
6489                                                 instr,
6490                                                 data);
6491
6492         if (!strcmp(tokens[tpos], "validate"))
6493                 return instr_hdr_validate_translate(p,
6494                                                     action,
6495                                                     &tokens[tpos],
6496                                                     n_tokens - tpos,
6497                                                     instr,
6498                                                     data);
6499
6500         if (!strcmp(tokens[tpos], "invalidate"))
6501                 return instr_hdr_invalidate_translate(p,
6502                                                       action,
6503                                                       &tokens[tpos],
6504                                                       n_tokens - tpos,
6505                                                       instr,
6506                                                       data);
6507
6508         if (!strcmp(tokens[tpos], "mov"))
6509                 return instr_mov_translate(p,
6510                                            action,
6511                                            &tokens[tpos],
6512                                            n_tokens - tpos,
6513                                            instr,
6514                                            data);
6515
6516         if (!strcmp(tokens[tpos], "add"))
6517                 return instr_alu_add_translate(p,
6518                                                action,
6519                                                &tokens[tpos],
6520                                                n_tokens - tpos,
6521                                                instr,
6522                                                data);
6523
6524         if (!strcmp(tokens[tpos], "sub"))
6525                 return instr_alu_sub_translate(p,
6526                                                action,
6527                                                &tokens[tpos],
6528                                                n_tokens - tpos,
6529                                                instr,
6530                                                data);
6531
6532         if (!strcmp(tokens[tpos], "ckadd"))
6533                 return instr_alu_ckadd_translate(p,
6534                                                  action,
6535                                                  &tokens[tpos],
6536                                                  n_tokens - tpos,
6537                                                  instr,
6538                                                  data);
6539
6540         if (!strcmp(tokens[tpos], "cksub"))
6541                 return instr_alu_cksub_translate(p,
6542                                                  action,
6543                                                  &tokens[tpos],
6544                                                  n_tokens - tpos,
6545                                                  instr,
6546                                                  data);
6547
6548         if (!strcmp(tokens[tpos], "and"))
6549                 return instr_alu_and_translate(p,
6550                                                action,
6551                                                &tokens[tpos],
6552                                                n_tokens - tpos,
6553                                                instr,
6554                                                data);
6555
6556         if (!strcmp(tokens[tpos], "or"))
6557                 return instr_alu_or_translate(p,
6558                                               action,
6559                                               &tokens[tpos],
6560                                               n_tokens - tpos,
6561                                               instr,
6562                                               data);
6563
6564         if (!strcmp(tokens[tpos], "xor"))
6565                 return instr_alu_xor_translate(p,
6566                                                action,
6567                                                &tokens[tpos],
6568                                                n_tokens - tpos,
6569                                                instr,
6570                                                data);
6571
6572         if (!strcmp(tokens[tpos], "shl"))
6573                 return instr_alu_shl_translate(p,
6574                                                action,
6575                                                &tokens[tpos],
6576                                                n_tokens - tpos,
6577                                                instr,
6578                                                data);
6579
6580         if (!strcmp(tokens[tpos], "shr"))
6581                 return instr_alu_shr_translate(p,
6582                                                action,
6583                                                &tokens[tpos],
6584                                                n_tokens - tpos,
6585                                                instr,
6586                                                data);
6587
6588         if (!strcmp(tokens[tpos], "regprefetch"))
6589                 return instr_regprefetch_translate(p,
6590                                                    action,
6591                                                    &tokens[tpos],
6592                                                    n_tokens - tpos,
6593                                                    instr,
6594                                                    data);
6595
6596         if (!strcmp(tokens[tpos], "regrd"))
6597                 return instr_regrd_translate(p,
6598                                              action,
6599                                              &tokens[tpos],
6600                                              n_tokens - tpos,
6601                                              instr,
6602                                              data);
6603
6604         if (!strcmp(tokens[tpos], "regwr"))
6605                 return instr_regwr_translate(p,
6606                                              action,
6607                                              &tokens[tpos],
6608                                              n_tokens - tpos,
6609                                              instr,
6610                                              data);
6611
6612         if (!strcmp(tokens[tpos], "regadd"))
6613                 return instr_regadd_translate(p,
6614                                               action,
6615                                               &tokens[tpos],
6616                                               n_tokens - tpos,
6617                                               instr,
6618                                               data);
6619
6620         if (!strcmp(tokens[tpos], "metprefetch"))
6621                 return instr_metprefetch_translate(p,
6622                                                    action,
6623                                                    &tokens[tpos],
6624                                                    n_tokens - tpos,
6625                                                    instr,
6626                                                    data);
6627
6628         if (!strcmp(tokens[tpos], "meter"))
6629                 return instr_meter_translate(p,
6630                                              action,
6631                                              &tokens[tpos],
6632                                              n_tokens - tpos,
6633                                              instr,
6634                                              data);
6635
6636         if (!strcmp(tokens[tpos], "table"))
6637                 return instr_table_translate(p,
6638                                              action,
6639                                              &tokens[tpos],
6640                                              n_tokens - tpos,
6641                                              instr,
6642                                              data);
6643
6644         if (!strcmp(tokens[tpos], "learn"))
6645                 return instr_learn_translate(p,
6646                                              action,
6647                                              &tokens[tpos],
6648                                              n_tokens - tpos,
6649                                              instr,
6650                                              data);
6651
6652         if (!strcmp(tokens[tpos], "forget"))
6653                 return instr_forget_translate(p,
6654                                               action,
6655                                               &tokens[tpos],
6656                                               n_tokens - tpos,
6657                                               instr,
6658                                               data);
6659
6660         if (!strcmp(tokens[tpos], "extern"))
6661                 return instr_extern_translate(p,
6662                                               action,
6663                                               &tokens[tpos],
6664                                               n_tokens - tpos,
6665                                               instr,
6666                                               data);
6667
6668         if (!strcmp(tokens[tpos], "jmp"))
6669                 return instr_jmp_translate(p,
6670                                            action,
6671                                            &tokens[tpos],
6672                                            n_tokens - tpos,
6673                                            instr,
6674                                            data);
6675
6676         if (!strcmp(tokens[tpos], "jmpv"))
6677                 return instr_jmp_valid_translate(p,
6678                                                  action,
6679                                                  &tokens[tpos],
6680                                                  n_tokens - tpos,
6681                                                  instr,
6682                                                  data);
6683
6684         if (!strcmp(tokens[tpos], "jmpnv"))
6685                 return instr_jmp_invalid_translate(p,
6686                                                    action,
6687                                                    &tokens[tpos],
6688                                                    n_tokens - tpos,
6689                                                    instr,
6690                                                    data);
6691
6692         if (!strcmp(tokens[tpos], "jmph"))
6693                 return instr_jmp_hit_translate(p,
6694                                                action,
6695                                                &tokens[tpos],
6696                                                n_tokens - tpos,
6697                                                instr,
6698                                                data);
6699
6700         if (!strcmp(tokens[tpos], "jmpnh"))
6701                 return instr_jmp_miss_translate(p,
6702                                                 action,
6703                                                 &tokens[tpos],
6704                                                 n_tokens - tpos,
6705                                                 instr,
6706                                                 data);
6707
6708         if (!strcmp(tokens[tpos], "jmpa"))
6709                 return instr_jmp_action_hit_translate(p,
6710                                                       action,
6711                                                       &tokens[tpos],
6712                                                       n_tokens - tpos,
6713                                                       instr,
6714                                                       data);
6715
6716         if (!strcmp(tokens[tpos], "jmpna"))
6717                 return instr_jmp_action_miss_translate(p,
6718                                                        action,
6719                                                        &tokens[tpos],
6720                                                        n_tokens - tpos,
6721                                                        instr,
6722                                                        data);
6723
6724         if (!strcmp(tokens[tpos], "jmpeq"))
6725                 return instr_jmp_eq_translate(p,
6726                                               action,
6727                                               &tokens[tpos],
6728                                               n_tokens - tpos,
6729                                               instr,
6730                                               data);
6731
6732         if (!strcmp(tokens[tpos], "jmpneq"))
6733                 return instr_jmp_neq_translate(p,
6734                                                action,
6735                                                &tokens[tpos],
6736                                                n_tokens - tpos,
6737                                                instr,
6738                                                data);
6739
6740         if (!strcmp(tokens[tpos], "jmplt"))
6741                 return instr_jmp_lt_translate(p,
6742                                               action,
6743                                               &tokens[tpos],
6744                                               n_tokens - tpos,
6745                                               instr,
6746                                               data);
6747
6748         if (!strcmp(tokens[tpos], "jmpgt"))
6749                 return instr_jmp_gt_translate(p,
6750                                               action,
6751                                               &tokens[tpos],
6752                                               n_tokens - tpos,
6753                                               instr,
6754                                               data);
6755
6756         if (!strcmp(tokens[tpos], "return"))
6757                 return instr_return_translate(p,
6758                                               action,
6759                                               &tokens[tpos],
6760                                               n_tokens - tpos,
6761                                               instr,
6762                                               data);
6763
6764         CHECK(0, EINVAL);
6765 }
6766
6767 static struct instruction_data *
6768 label_find(struct instruction_data *data, uint32_t n, const char *label)
6769 {
6770         uint32_t i;
6771
6772         for (i = 0; i < n; i++)
6773                 if (!strcmp(label, data[i].label))
6774                         return &data[i];
6775
6776         return NULL;
6777 }
6778
6779 static uint32_t
6780 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6781 {
6782         uint32_t count = 0, i;
6783
6784         if (!label[0])
6785                 return 0;
6786
6787         for (i = 0; i < n; i++)
6788                 if (!strcmp(label, data[i].jmp_label))
6789                         count++;
6790
6791         return count;
6792 }
6793
6794 static int
6795 instr_label_check(struct instruction_data *instruction_data,
6796                   uint32_t n_instructions)
6797 {
6798         uint32_t i;
6799
6800         /* Check that all instruction labels are unique. */
6801         for (i = 0; i < n_instructions; i++) {
6802                 struct instruction_data *data = &instruction_data[i];
6803                 char *label = data->label;
6804                 uint32_t j;
6805
6806                 if (!label[0])
6807                         continue;
6808
6809                 for (j = i + 1; j < n_instructions; j++)
6810                         CHECK(strcmp(label, data[j].label), EINVAL);
6811         }
6812
6813         /* Get users for each instruction label. */
6814         for (i = 0; i < n_instructions; i++) {
6815                 struct instruction_data *data = &instruction_data[i];
6816                 char *label = data->label;
6817
6818                 data->n_users = label_is_used(instruction_data,
6819                                               n_instructions,
6820                                               label);
6821         }
6822
6823         return 0;
6824 }
6825
6826 static int
6827 instr_jmp_resolve(struct instruction *instructions,
6828                   struct instruction_data *instruction_data,
6829                   uint32_t n_instructions)
6830 {
6831         uint32_t i;
6832
6833         for (i = 0; i < n_instructions; i++) {
6834                 struct instruction *instr = &instructions[i];
6835                 struct instruction_data *data = &instruction_data[i];
6836                 struct instruction_data *found;
6837
6838                 if (!instruction_is_jmp(instr))
6839                         continue;
6840
6841                 found = label_find(instruction_data,
6842                                    n_instructions,
6843                                    data->jmp_label);
6844                 CHECK(found, EINVAL);
6845
6846                 instr->jmp.ip = &instructions[found - instruction_data];
6847         }
6848
6849         return 0;
6850 }
6851
6852 static int
6853 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6854              struct action *a,
6855              struct instruction *instr,
6856              struct instruction_data *data __rte_unused,
6857              uint32_t n_instructions)
6858 {
6859         if (!a) {
6860                 enum instruction_type type;
6861                 uint32_t i;
6862
6863                 /* Check that the first instruction is rx. */
6864                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6865
6866                 /* Check that there is at least one tx instruction. */
6867                 for (i = 0; i < n_instructions; i++) {
6868                         type = instr[i].type;
6869
6870                         if (instruction_is_tx(type))
6871                                 break;
6872                 }
6873                 CHECK(i < n_instructions, EINVAL);
6874
6875                 /* Check that the last instruction is either tx or unconditional
6876                  * jump.
6877                  */
6878                 type = instr[n_instructions - 1].type;
6879                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6880         }
6881
6882         if (a) {
6883                 enum instruction_type type;
6884                 uint32_t i;
6885
6886                 /* Check that there is at least one return or tx instruction. */
6887                 for (i = 0; i < n_instructions; i++) {
6888                         type = instr[i].type;
6889
6890                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6891                                 break;
6892                 }
6893                 CHECK(i < n_instructions, EINVAL);
6894         }
6895
6896         return 0;
6897 }
6898
6899 static uint32_t
6900 instr_compact(struct instruction *instructions,
6901               struct instruction_data *instruction_data,
6902               uint32_t n_instructions)
6903 {
6904         uint32_t i, pos = 0;
6905
6906         /* Eliminate the invalid instructions that have been optimized out. */
6907         for (i = 0; i < n_instructions; i++) {
6908                 struct instruction *instr = &instructions[i];
6909                 struct instruction_data *data = &instruction_data[i];
6910
6911                 if (data->invalid)
6912                         continue;
6913
6914                 if (i != pos) {
6915                         memcpy(&instructions[pos], instr, sizeof(*instr));
6916                         memcpy(&instruction_data[pos], data, sizeof(*data));
6917                 }
6918
6919                 pos++;
6920         }
6921
6922         return pos;
6923 }
6924
6925 static int
6926 instr_pattern_extract_many_search(struct instruction *instr,
6927                                   struct instruction_data *data,
6928                                   uint32_t n_instr,
6929                                   uint32_t *n_pattern_instr)
6930 {
6931         uint32_t i;
6932
6933         for (i = 0; i < n_instr; i++) {
6934                 if (data[i].invalid)
6935                         break;
6936
6937                 if (instr[i].type != INSTR_HDR_EXTRACT)
6938                         break;
6939
6940                 if (i == RTE_DIM(instr->io.hdr.header_id))
6941                         break;
6942
6943                 if (i && data[i].n_users)
6944                         break;
6945         }
6946
6947         if (i < 2)
6948                 return 0;
6949
6950         *n_pattern_instr = i;
6951         return 1;
6952 }
6953
6954 static void
6955 instr_pattern_extract_many_replace(struct instruction *instr,
6956                                    struct instruction_data *data,
6957                                    uint32_t n_instr)
6958 {
6959         uint32_t i;
6960
6961         for (i = 1; i < n_instr; i++) {
6962                 instr[0].type++;
6963                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6964                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6965                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6966
6967                 data[i].invalid = 1;
6968         }
6969 }
6970
6971 static uint32_t
6972 instr_pattern_extract_many_optimize(struct instruction *instructions,
6973                                     struct instruction_data *instruction_data,
6974                                     uint32_t n_instructions)
6975 {
6976         uint32_t i;
6977
6978         for (i = 0; i < n_instructions; ) {
6979                 struct instruction *instr = &instructions[i];
6980                 struct instruction_data *data = &instruction_data[i];
6981                 uint32_t n_instr = 0;
6982                 int detected;
6983
6984                 /* Extract many. */
6985                 detected = instr_pattern_extract_many_search(instr,
6986                                                              data,
6987                                                              n_instructions - i,
6988                                                              &n_instr);
6989                 if (detected) {
6990                         instr_pattern_extract_many_replace(instr,
6991                                                            data,
6992                                                            n_instr);
6993                         i += n_instr;
6994                         continue;
6995                 }
6996
6997                 /* No pattern starting at the current instruction. */
6998                 i++;
6999         }
7000
7001         /* Eliminate the invalid instructions that have been optimized out. */
7002         n_instructions = instr_compact(instructions,
7003                                        instruction_data,
7004                                        n_instructions);
7005
7006         return n_instructions;
7007 }
7008
7009 static int
7010 instr_pattern_emit_many_tx_search(struct instruction *instr,
7011                                   struct instruction_data *data,
7012                                   uint32_t n_instr,
7013                                   uint32_t *n_pattern_instr)
7014 {
7015         uint32_t i;
7016
7017         for (i = 0; i < n_instr; i++) {
7018                 if (data[i].invalid)
7019                         break;
7020
7021                 if (instr[i].type != INSTR_HDR_EMIT)
7022                         break;
7023
7024                 if (i == RTE_DIM(instr->io.hdr.header_id))
7025                         break;
7026
7027                 if (i && data[i].n_users)
7028                         break;
7029         }
7030
7031         if (!i)
7032                 return 0;
7033
7034         if (!instruction_is_tx(instr[i].type))
7035                 return 0;
7036
7037         if (data[i].n_users)
7038                 return 0;
7039
7040         i++;
7041
7042         *n_pattern_instr = i;
7043         return 1;
7044 }
7045
7046 static void
7047 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7048                                    struct instruction_data *data,
7049                                    uint32_t n_instr)
7050 {
7051         uint32_t i;
7052
7053         /* Any emit instruction in addition to the first one. */
7054         for (i = 1; i < n_instr - 1; i++) {
7055                 instr[0].type++;
7056                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7057                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7058                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7059
7060                 data[i].invalid = 1;
7061         }
7062
7063         /* The TX instruction is the last one in the pattern. */
7064         instr[0].type++;
7065         instr[0].io.io.offset = instr[i].io.io.offset;
7066         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7067         data[i].invalid = 1;
7068 }
7069
7070 static uint32_t
7071 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7072                                     struct instruction_data *instruction_data,
7073                                     uint32_t n_instructions)
7074 {
7075         uint32_t i;
7076
7077         for (i = 0; i < n_instructions; ) {
7078                 struct instruction *instr = &instructions[i];
7079                 struct instruction_data *data = &instruction_data[i];
7080                 uint32_t n_instr = 0;
7081                 int detected;
7082
7083                 /* Emit many + TX. */
7084                 detected = instr_pattern_emit_many_tx_search(instr,
7085                                                              data,
7086                                                              n_instructions - i,
7087                                                              &n_instr);
7088                 if (detected) {
7089                         instr_pattern_emit_many_tx_replace(instr,
7090                                                            data,
7091                                                            n_instr);
7092                         i += n_instr;
7093                         continue;
7094                 }
7095
7096                 /* No pattern starting at the current instruction. */
7097                 i++;
7098         }
7099
7100         /* Eliminate the invalid instructions that have been optimized out. */
7101         n_instructions = instr_compact(instructions,
7102                                        instruction_data,
7103                                        n_instructions);
7104
7105         return n_instructions;
7106 }
7107
7108 static uint32_t
7109 action_arg_src_mov_count(struct action *a,
7110                          uint32_t arg_id,
7111                          struct instruction *instructions,
7112                          struct instruction_data *instruction_data,
7113                          uint32_t n_instructions);
7114
7115 static int
7116 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7117                                       struct action *a,
7118                                       struct instruction *instr,
7119                                       struct instruction_data *data,
7120                                       uint32_t n_instr,
7121                                       struct instruction *instructions,
7122                                       struct instruction_data *instruction_data,
7123                                       uint32_t n_instructions,
7124                                       uint32_t *n_pattern_instr)
7125 {
7126         struct header *h;
7127         uint32_t src_field_id, i, j;
7128
7129         /* Prerequisites. */
7130         if (!a || !a->st)
7131                 return 0;
7132
7133         /* First instruction: MOV_HM. */
7134         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7135                 return 0;
7136
7137         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7138         if (!h || h->st->var_size)
7139                 return 0;
7140
7141         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7142                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7143                         break;
7144
7145         if (src_field_id == a->st->n_fields)
7146                 return 0;
7147
7148         if (instr[0].mov.dst.offset ||
7149             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7150             instr[0].mov.src.struct_id ||
7151             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7152             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7153                 return 0;
7154
7155         if ((n_instr < h->st->n_fields + 1) ||
7156              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7157                 return 0;
7158
7159         /* Subsequent instructions: MOV_HM. */
7160         for (i = 1; i < h->st->n_fields; i++)
7161                 if (data[i].invalid ||
7162                     data[i].n_users ||
7163                     (instr[i].type != INSTR_MOV_HM) ||
7164                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7165                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7166                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7167                     instr[i].mov.src.struct_id ||
7168                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7169                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7170                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7171                         return 0;
7172
7173         /* Last instruction: HDR_VALIDATE. */
7174         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7175             (instr[i].valid.header_id != h->id))
7176                 return 0;
7177
7178         /* Check that none of the action args that are used as source for this
7179          * DMA transfer are not used as source in any other mov instruction.
7180          */
7181         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7182                 uint32_t n_users;
7183
7184                 n_users = action_arg_src_mov_count(a,
7185                                                    j,
7186                                                    instructions,
7187                                                    instruction_data,
7188                                                    n_instructions);
7189                 if (n_users > 1)
7190                         return 0;
7191         }
7192
7193         *n_pattern_instr = 1 + i;
7194         return 1;
7195 }
7196
7197 static void
7198 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7199                                        struct action *a,
7200                                        struct instruction *instr,
7201                                        struct instruction_data *data,
7202                                        uint32_t n_instr)
7203 {
7204         struct header *h;
7205         uint32_t src_field_id, src_offset, i;
7206
7207         /* Read from the instructions before they are modified. */
7208         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7209         if (!h)
7210                 return;
7211
7212         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7213                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7214                         break;
7215
7216         if (src_field_id == a->st->n_fields)
7217                 return;
7218
7219         src_offset = instr[0].mov.src.offset;
7220
7221         /* Modify the instructions. */
7222         instr[0].type = INSTR_DMA_HT;
7223         instr[0].dma.dst.header_id[0] = h->id;
7224         instr[0].dma.dst.struct_id[0] = h->struct_id;
7225         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7226         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7227
7228         for (i = 1; i < n_instr; i++)
7229                 data[i].invalid = 1;
7230
7231         /* Update the endianness of the action arguments to header endianness. */
7232         for (i = 0; i < h->st->n_fields; i++)
7233                 a->args_endianness[src_field_id + i] = 1;
7234 }
7235
7236 static uint32_t
7237 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7238                                         struct action *a,
7239                                         struct instruction *instructions,
7240                                         struct instruction_data *instruction_data,
7241                                         uint32_t n_instructions)
7242 {
7243         uint32_t i;
7244
7245         if (!a || !a->st)
7246                 return n_instructions;
7247
7248         for (i = 0; i < n_instructions; ) {
7249                 struct instruction *instr = &instructions[i];
7250                 struct instruction_data *data = &instruction_data[i];
7251                 uint32_t n_instr = 0;
7252                 int detected;
7253
7254                 /* Mov all + validate. */
7255                 detected = instr_pattern_mov_all_validate_search(p,
7256                                                                  a,
7257                                                                  instr,
7258                                                                  data,
7259                                                                  n_instructions - i,
7260                                                                  instructions,
7261                                                                  instruction_data,
7262                                                                  n_instructions,
7263                                                                  &n_instr);
7264                 if (detected) {
7265                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7266                         i += n_instr;
7267                         continue;
7268                 }
7269
7270                 /* No pattern starting at the current instruction. */
7271                 i++;
7272         }
7273
7274         /* Eliminate the invalid instructions that have been optimized out. */
7275         n_instructions = instr_compact(instructions,
7276                                        instruction_data,
7277                                        n_instructions);
7278
7279         return n_instructions;
7280 }
7281
7282 static int
7283 instr_pattern_dma_many_search(struct instruction *instr,
7284                               struct instruction_data *data,
7285                               uint32_t n_instr,
7286                               uint32_t *n_pattern_instr)
7287 {
7288         uint32_t i;
7289
7290         for (i = 0; i < n_instr; i++) {
7291                 if (data[i].invalid)
7292                         break;
7293
7294                 if (instr[i].type != INSTR_DMA_HT)
7295                         break;
7296
7297                 if (i == RTE_DIM(instr->dma.dst.header_id))
7298                         break;
7299
7300                 if (i && data[i].n_users)
7301                         break;
7302         }
7303
7304         if (i < 2)
7305                 return 0;
7306
7307         *n_pattern_instr = i;
7308         return 1;
7309 }
7310
7311 static void
7312 instr_pattern_dma_many_replace(struct instruction *instr,
7313                                struct instruction_data *data,
7314                                uint32_t n_instr)
7315 {
7316         uint32_t i;
7317
7318         for (i = 1; i < n_instr; i++) {
7319                 instr[0].type++;
7320                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7321                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7322                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7323                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7324
7325                 data[i].invalid = 1;
7326         }
7327 }
7328
7329 static uint32_t
7330 instr_pattern_dma_many_optimize(struct instruction *instructions,
7331                struct instruction_data *instruction_data,
7332                uint32_t n_instructions)
7333 {
7334         uint32_t i;
7335
7336         for (i = 0; i < n_instructions; ) {
7337                 struct instruction *instr = &instructions[i];
7338                 struct instruction_data *data = &instruction_data[i];
7339                 uint32_t n_instr = 0;
7340                 int detected;
7341
7342                 /* DMA many. */
7343                 detected = instr_pattern_dma_many_search(instr,
7344                                                          data,
7345                                                          n_instructions - i,
7346                                                          &n_instr);
7347                 if (detected) {
7348                         instr_pattern_dma_many_replace(instr, data, n_instr);
7349                         i += n_instr;
7350                         continue;
7351                 }
7352
7353                 /* No pattern starting at the current instruction. */
7354                 i++;
7355         }
7356
7357         /* Eliminate the invalid instructions that have been optimized out. */
7358         n_instructions = instr_compact(instructions,
7359                                        instruction_data,
7360                                        n_instructions);
7361
7362         return n_instructions;
7363 }
7364
7365 static uint32_t
7366 instr_optimize(struct rte_swx_pipeline *p,
7367                struct action *a,
7368                struct instruction *instructions,
7369                struct instruction_data *instruction_data,
7370                uint32_t n_instructions)
7371 {
7372         /* Extract many. */
7373         n_instructions = instr_pattern_extract_many_optimize(instructions,
7374                                                              instruction_data,
7375                                                              n_instructions);
7376
7377         /* Emit many + TX. */
7378         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7379                                                              instruction_data,
7380                                                              n_instructions);
7381
7382         /* Mov all + validate. */
7383         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7384                                                                  a,
7385                                                                  instructions,
7386                                                                  instruction_data,
7387                                                                  n_instructions);
7388
7389         /* DMA many. */
7390         n_instructions = instr_pattern_dma_many_optimize(instructions,
7391                                                          instruction_data,
7392                                                          n_instructions);
7393
7394         return n_instructions;
7395 }
7396
7397 static int
7398 instruction_config(struct rte_swx_pipeline *p,
7399                    struct action *a,
7400                    const char **instructions,
7401                    uint32_t n_instructions)
7402 {
7403         struct instruction *instr = NULL;
7404         struct instruction_data *data = NULL;
7405         int err = 0;
7406         uint32_t i;
7407
7408         CHECK(n_instructions, EINVAL);
7409         CHECK(instructions, EINVAL);
7410         for (i = 0; i < n_instructions; i++)
7411                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7412
7413         /* Memory allocation. */
7414         instr = calloc(n_instructions, sizeof(struct instruction));
7415         if (!instr) {
7416                 err = -ENOMEM;
7417                 goto error;
7418         }
7419
7420         data = calloc(n_instructions, sizeof(struct instruction_data));
7421         if (!data) {
7422                 err = -ENOMEM;
7423                 goto error;
7424         }
7425
7426         for (i = 0; i < n_instructions; i++) {
7427                 char *string = strdup(instructions[i]);
7428                 if (!string) {
7429                         err = -ENOMEM;
7430                         goto error;
7431                 }
7432
7433                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7434                 if (err) {
7435                         free(string);
7436                         goto error;
7437                 }
7438
7439                 free(string);
7440         }
7441
7442         err = instr_label_check(data, n_instructions);
7443         if (err)
7444                 goto error;
7445
7446         err = instr_verify(p, a, instr, data, n_instructions);
7447         if (err)
7448                 goto error;
7449
7450         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7451
7452         err = instr_jmp_resolve(instr, data, n_instructions);
7453         if (err)
7454                 goto error;
7455
7456         if (a) {
7457                 a->instructions = instr;
7458                 a->n_instructions = n_instructions;
7459         } else {
7460                 p->instructions = instr;
7461                 p->n_instructions = n_instructions;
7462         }
7463
7464         free(data);
7465         return 0;
7466
7467 error:
7468         free(data);
7469         free(instr);
7470         return err;
7471 }
7472
7473 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7474
7475 static instr_exec_t instruction_table[] = {
7476         [INSTR_RX] = instr_rx_exec,
7477         [INSTR_TX] = instr_tx_exec,
7478         [INSTR_TX_I] = instr_tx_i_exec,
7479
7480         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7481         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7482         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7483         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7484         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7485         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7486         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7487         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7488         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7489         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7490
7491         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7492         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7493         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7494         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7495         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7496         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7497         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7498         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7499         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7500
7501         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7502         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7503
7504         [INSTR_MOV] = instr_mov_exec,
7505         [INSTR_MOV_MH] = instr_mov_mh_exec,
7506         [INSTR_MOV_HM] = instr_mov_hm_exec,
7507         [INSTR_MOV_HH] = instr_mov_hh_exec,
7508         [INSTR_MOV_I] = instr_mov_i_exec,
7509
7510         [INSTR_DMA_HT] = instr_dma_ht_exec,
7511         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7512         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7513         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7514         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7515         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7516         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7517         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7518
7519         [INSTR_ALU_ADD] = instr_alu_add_exec,
7520         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7521         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7522         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7523         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7524         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7525
7526         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7527         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7528         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7529         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7530         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7531         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7532
7533         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7534         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7535         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7536         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7537
7538         [INSTR_ALU_AND] = instr_alu_and_exec,
7539         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7540         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7541         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7542         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7543
7544         [INSTR_ALU_OR] = instr_alu_or_exec,
7545         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7546         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7547         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7548         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7549
7550         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7551         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7552         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7553         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7554         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7555
7556         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7557         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7558         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7559         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7560         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7561         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7562
7563         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7564         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7565         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7566         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7567         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7568         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7569
7570         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7571         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7572         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7573
7574         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7575         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7576         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7577         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7578         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7579         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7580
7581         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7582         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7583         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7584         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7585         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7586         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7587         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7588         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7589         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7590
7591         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7592         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7593         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7594         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7595         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7596         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7597         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7598         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7599         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7600
7601         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7602         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7603         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7604
7605         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7606         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7607         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7608         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7609         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7610         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7611         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7612         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7613         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7614         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7615         [INSTR_METER_IMM] = instr_meter_imm_exec,
7616         [INSTR_METER_IMI] = instr_meter_imi_exec,
7617
7618         [INSTR_TABLE] = instr_table_exec,
7619         [INSTR_SELECTOR] = instr_selector_exec,
7620         [INSTR_LEARNER] = instr_learner_exec,
7621         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7622         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7623         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7624         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7625
7626         [INSTR_JMP] = instr_jmp_exec,
7627         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7628         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7629         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7630         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7631         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7632         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7633
7634         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7635         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7636         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7637         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7638         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7639
7640         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7641         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7642         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7643         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7644         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7645
7646         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7647         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7648         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7649         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7650         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7651         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7652
7653         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7654         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7655         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7656         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7657         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7658         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7659
7660         [INSTR_RETURN] = instr_return_exec,
7661 };
7662
7663 static inline void
7664 instr_exec(struct rte_swx_pipeline *p)
7665 {
7666         struct thread *t = &p->threads[p->thread_id];
7667         struct instruction *ip = t->ip;
7668         instr_exec_t instr = instruction_table[ip->type];
7669
7670         instr(p);
7671 }
7672
7673 /*
7674  * Action.
7675  */
7676 static struct action *
7677 action_find(struct rte_swx_pipeline *p, const char *name)
7678 {
7679         struct action *elem;
7680
7681         if (!name)
7682                 return NULL;
7683
7684         TAILQ_FOREACH(elem, &p->actions, node)
7685                 if (strcmp(elem->name, name) == 0)
7686                         return elem;
7687
7688         return NULL;
7689 }
7690
7691 static struct action *
7692 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7693 {
7694         struct action *action = NULL;
7695
7696         TAILQ_FOREACH(action, &p->actions, node)
7697                 if (action->id == id)
7698                         return action;
7699
7700         return NULL;
7701 }
7702
7703 static struct field *
7704 action_field_find(struct action *a, const char *name)
7705 {
7706         return a->st ? struct_type_field_find(a->st, name) : NULL;
7707 }
7708
7709 static struct field *
7710 action_field_parse(struct action *action, const char *name)
7711 {
7712         if (name[0] != 't' || name[1] != '.')
7713                 return NULL;
7714
7715         return action_field_find(action, &name[2]);
7716 }
7717
7718 static int
7719 action_has_nbo_args(struct action *a)
7720 {
7721         uint32_t i;
7722
7723         /* Return if the action does not have any args. */
7724         if (!a->st)
7725                 return 0; /* FALSE */
7726
7727         for (i = 0; i < a->st->n_fields; i++)
7728                 if (a->args_endianness[i])
7729                         return 1; /* TRUE */
7730
7731         return 0; /* FALSE */
7732 }
7733
7734 static int
7735 action_does_learning(struct action *a)
7736 {
7737         uint32_t i;
7738
7739         for (i = 0; i < a->n_instructions; i++)
7740                 switch (a->instructions[i].type) {
7741                 case INSTR_LEARNER_LEARN:
7742                         return 1; /* TRUE */
7743
7744                 case INSTR_LEARNER_FORGET:
7745                         return 1; /* TRUE */
7746
7747                 default:
7748                         continue;
7749                 }
7750
7751         return 0; /* FALSE */
7752 }
7753
7754 int
7755 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7756                                const char *name,
7757                                const char *args_struct_type_name,
7758                                const char **instructions,
7759                                uint32_t n_instructions)
7760 {
7761         struct struct_type *args_struct_type = NULL;
7762         struct action *a;
7763         int err;
7764
7765         CHECK(p, EINVAL);
7766
7767         CHECK_NAME(name, EINVAL);
7768         CHECK(!action_find(p, name), EEXIST);
7769
7770         if (args_struct_type_name) {
7771                 CHECK_NAME(args_struct_type_name, EINVAL);
7772                 args_struct_type = struct_type_find(p, args_struct_type_name);
7773                 CHECK(args_struct_type, EINVAL);
7774                 CHECK(!args_struct_type->var_size, EINVAL);
7775         }
7776
7777         /* Node allocation. */
7778         a = calloc(1, sizeof(struct action));
7779         CHECK(a, ENOMEM);
7780         if (args_struct_type) {
7781                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7782                 if (!a->args_endianness) {
7783                         free(a);
7784                         CHECK(0, ENOMEM);
7785                 }
7786         }
7787
7788         /* Node initialization. */
7789         strcpy(a->name, name);
7790         a->st = args_struct_type;
7791         a->id = p->n_actions;
7792
7793         /* Instruction translation. */
7794         err = instruction_config(p, a, instructions, n_instructions);
7795         if (err) {
7796                 free(a->args_endianness);
7797                 free(a);
7798                 return err;
7799         }
7800
7801         /* Node add to tailq. */
7802         TAILQ_INSERT_TAIL(&p->actions, a, node);
7803         p->n_actions++;
7804
7805         return 0;
7806 }
7807
7808 static int
7809 action_build(struct rte_swx_pipeline *p)
7810 {
7811         struct action *action;
7812
7813         p->action_instructions = calloc(p->n_actions,
7814                                         sizeof(struct instruction *));
7815         CHECK(p->action_instructions, ENOMEM);
7816
7817         TAILQ_FOREACH(action, &p->actions, node)
7818                 p->action_instructions[action->id] = action->instructions;
7819
7820         return 0;
7821 }
7822
7823 static void
7824 action_build_free(struct rte_swx_pipeline *p)
7825 {
7826         free(p->action_instructions);
7827         p->action_instructions = NULL;
7828 }
7829
7830 static void
7831 action_free(struct rte_swx_pipeline *p)
7832 {
7833         action_build_free(p);
7834
7835         for ( ; ; ) {
7836                 struct action *action;
7837
7838                 action = TAILQ_FIRST(&p->actions);
7839                 if (!action)
7840                         break;
7841
7842                 TAILQ_REMOVE(&p->actions, action, node);
7843                 free(action->instructions);
7844                 free(action);
7845         }
7846 }
7847
7848 static uint32_t
7849 action_arg_src_mov_count(struct action *a,
7850                          uint32_t arg_id,
7851                          struct instruction *instructions,
7852                          struct instruction_data *instruction_data,
7853                          uint32_t n_instructions)
7854 {
7855         uint32_t offset, n_users = 0, i;
7856
7857         if (!a->st ||
7858             (arg_id >= a->st->n_fields) ||
7859             !instructions ||
7860             !instruction_data ||
7861             !n_instructions)
7862                 return 0;
7863
7864         offset = a->st->fields[arg_id].offset / 8;
7865
7866         for (i = 0; i < n_instructions; i++) {
7867                 struct instruction *instr = &instructions[i];
7868                 struct instruction_data *data = &instruction_data[i];
7869
7870                 if (data->invalid ||
7871                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7872                     instr->mov.src.struct_id ||
7873                     (instr->mov.src.offset != offset))
7874                         continue;
7875
7876                 n_users++;
7877         }
7878
7879         return n_users;
7880 }
7881
7882 /*
7883  * Table.
7884  */
7885 static struct table_type *
7886 table_type_find(struct rte_swx_pipeline *p, const char *name)
7887 {
7888         struct table_type *elem;
7889
7890         TAILQ_FOREACH(elem, &p->table_types, node)
7891                 if (strcmp(elem->name, name) == 0)
7892                         return elem;
7893
7894         return NULL;
7895 }
7896
7897 static struct table_type *
7898 table_type_resolve(struct rte_swx_pipeline *p,
7899                    const char *recommended_type_name,
7900                    enum rte_swx_table_match_type match_type)
7901 {
7902         struct table_type *elem;
7903
7904         /* Only consider the recommended type if the match type is correct. */
7905         if (recommended_type_name)
7906                 TAILQ_FOREACH(elem, &p->table_types, node)
7907                         if (!strcmp(elem->name, recommended_type_name) &&
7908                             (elem->match_type == match_type))
7909                                 return elem;
7910
7911         /* Ignore the recommended type and get the first element with this match
7912          * type.
7913          */
7914         TAILQ_FOREACH(elem, &p->table_types, node)
7915                 if (elem->match_type == match_type)
7916                         return elem;
7917
7918         return NULL;
7919 }
7920
7921 static struct table *
7922 table_find(struct rte_swx_pipeline *p, const char *name)
7923 {
7924         struct table *elem;
7925
7926         TAILQ_FOREACH(elem, &p->tables, node)
7927                 if (strcmp(elem->name, name) == 0)
7928                         return elem;
7929
7930         return NULL;
7931 }
7932
7933 static struct table *
7934 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7935 {
7936         struct table *table = NULL;
7937
7938         TAILQ_FOREACH(table, &p->tables, node)
7939                 if (table->id == id)
7940                         return table;
7941
7942         return NULL;
7943 }
7944
7945 int
7946 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7947                                      const char *name,
7948                                      enum rte_swx_table_match_type match_type,
7949                                      struct rte_swx_table_ops *ops)
7950 {
7951         struct table_type *elem;
7952
7953         CHECK(p, EINVAL);
7954
7955         CHECK_NAME(name, EINVAL);
7956         CHECK(!table_type_find(p, name), EEXIST);
7957
7958         CHECK(ops, EINVAL);
7959         CHECK(ops->create, EINVAL);
7960         CHECK(ops->lkp, EINVAL);
7961         CHECK(ops->free, EINVAL);
7962
7963         /* Node allocation. */
7964         elem = calloc(1, sizeof(struct table_type));
7965         CHECK(elem, ENOMEM);
7966
7967         /* Node initialization. */
7968         strcpy(elem->name, name);
7969         elem->match_type = match_type;
7970         memcpy(&elem->ops, ops, sizeof(*ops));
7971
7972         /* Node add to tailq. */
7973         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7974
7975         return 0;
7976 }
7977
7978 static int
7979 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7980                          uint32_t n_fields,
7981                          enum rte_swx_table_match_type *match_type)
7982 {
7983         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7984
7985         for (i = 0; i < n_fields; i++) {
7986                 struct rte_swx_match_field_params  *f = &fields[i];
7987
7988                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7989                         n_fields_em++;
7990
7991                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7992                         n_fields_lpm++;
7993         }
7994
7995         if ((n_fields_lpm > 1) ||
7996             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7997                 return -EINVAL;
7998
7999         *match_type = (n_fields_em == n_fields) ?
8000                        RTE_SWX_TABLE_MATCH_EXACT :
8001                        RTE_SWX_TABLE_MATCH_WILDCARD;
8002
8003         return 0;
8004 }
8005
8006 static int
8007 table_match_fields_check(struct rte_swx_pipeline *p,
8008                          struct rte_swx_pipeline_table_params *params,
8009                          struct header **header)
8010 {
8011         struct header *h0 = NULL;
8012         struct field *hf, *mf;
8013         uint32_t *offset = NULL, i;
8014         int status = 0;
8015
8016         /* Return if no match fields. */
8017         if (!params->n_fields) {
8018                 if (params->fields) {
8019                         status = -EINVAL;
8020                         goto end;
8021                 }
8022
8023                 if (header)
8024                         *header = NULL;
8025
8026                 return 0;
8027         }
8028
8029         /* Memory allocation. */
8030         offset = calloc(params->n_fields, sizeof(uint32_t));
8031         if (!offset) {
8032                 status = -ENOMEM;
8033                 goto end;
8034         }
8035
8036         /* Check that all the match fields belong to either the same header or
8037          * to the meta-data.
8038          */
8039         hf = header_field_parse(p, params->fields[0].name, &h0);
8040         mf = metadata_field_parse(p, params->fields[0].name);
8041         if ((!hf && !mf) || (hf && hf->var_size)) {
8042                 status = -EINVAL;
8043                 goto end;
8044         }
8045
8046         offset[0] = h0 ? hf->offset : mf->offset;
8047
8048         for (i = 1; i < params->n_fields; i++)
8049                 if (h0) {
8050                         struct header *h;
8051
8052                         hf = header_field_parse(p, params->fields[i].name, &h);
8053                         if (!hf || (h->id != h0->id) || hf->var_size) {
8054                                 status = -EINVAL;
8055                                 goto end;
8056                         }
8057
8058                         offset[i] = hf->offset;
8059                 } else {
8060                         mf = metadata_field_parse(p, params->fields[i].name);
8061                         if (!mf) {
8062                                 status = -EINVAL;
8063                                 goto end;
8064                         }
8065
8066                         offset[i] = mf->offset;
8067                 }
8068
8069         /* Check that there are no duplicated match fields. */
8070         for (i = 0; i < params->n_fields; i++) {
8071                 uint32_t j;
8072
8073                 for (j = 0; j < i; j++)
8074                         if (offset[j] == offset[i]) {
8075                                 status = -EINVAL;
8076                                 goto end;
8077                         }
8078         }
8079
8080         /* Return. */
8081         if (header)
8082                 *header = h0;
8083
8084 end:
8085         free(offset);
8086         return status;
8087 }
8088
8089 int
8090 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8091                               const char *name,
8092                               struct rte_swx_pipeline_table_params *params,
8093                               const char *recommended_table_type_name,
8094                               const char *args,
8095                               uint32_t size)
8096 {
8097         struct table_type *type;
8098         struct table *t;
8099         struct action *default_action;
8100         struct header *header = NULL;
8101         uint32_t action_data_size_max = 0, i;
8102         int status = 0;
8103
8104         CHECK(p, EINVAL);
8105
8106         CHECK_NAME(name, EINVAL);
8107         CHECK(!table_find(p, name), EEXIST);
8108         CHECK(!selector_find(p, name), EEXIST);
8109         CHECK(!learner_find(p, name), EEXIST);
8110
8111         CHECK(params, EINVAL);
8112
8113         /* Match checks. */
8114         status = table_match_fields_check(p, params, &header);
8115         if (status)
8116                 return status;
8117
8118         /* Action checks. */
8119         CHECK(params->n_actions, EINVAL);
8120         CHECK(params->action_names, EINVAL);
8121         for (i = 0; i < params->n_actions; i++) {
8122                 const char *action_name = params->action_names[i];
8123                 struct action *a;
8124                 uint32_t action_data_size;
8125
8126                 CHECK_NAME(action_name, EINVAL);
8127
8128                 a = action_find(p, action_name);
8129                 CHECK(a, EINVAL);
8130                 CHECK(!action_does_learning(a), EINVAL);
8131
8132                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8133                 if (action_data_size > action_data_size_max)
8134                         action_data_size_max = action_data_size;
8135         }
8136
8137         CHECK_NAME(params->default_action_name, EINVAL);
8138         for (i = 0; i < p->n_actions; i++)
8139                 if (!strcmp(params->action_names[i],
8140                             params->default_action_name))
8141                         break;
8142         CHECK(i < params->n_actions, EINVAL);
8143         default_action = action_find(p, params->default_action_name);
8144         CHECK((default_action->st && params->default_action_data) ||
8145               !params->default_action_data, EINVAL);
8146
8147         /* Table type checks. */
8148         if (recommended_table_type_name)
8149                 CHECK_NAME(recommended_table_type_name, EINVAL);
8150
8151         if (params->n_fields) {
8152                 enum rte_swx_table_match_type match_type;
8153
8154                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8155                 if (status)
8156                         return status;
8157
8158                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8159                 CHECK(type, EINVAL);
8160         } else {
8161                 type = NULL;
8162         }
8163
8164         /* Memory allocation. */
8165         t = calloc(1, sizeof(struct table));
8166         CHECK(t, ENOMEM);
8167
8168         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8169         if (!t->fields) {
8170                 free(t);
8171                 CHECK(0, ENOMEM);
8172         }
8173
8174         t->actions = calloc(params->n_actions, sizeof(struct action *));
8175         if (!t->actions) {
8176                 free(t->fields);
8177                 free(t);
8178                 CHECK(0, ENOMEM);
8179         }
8180
8181         if (action_data_size_max) {
8182                 t->default_action_data = calloc(1, action_data_size_max);
8183                 if (!t->default_action_data) {
8184                         free(t->actions);
8185                         free(t->fields);
8186                         free(t);
8187                         CHECK(0, ENOMEM);
8188                 }
8189         }
8190
8191         /* Node initialization. */
8192         strcpy(t->name, name);
8193         if (args && args[0])
8194                 strcpy(t->args, args);
8195         t->type = type;
8196
8197         for (i = 0; i < params->n_fields; i++) {
8198                 struct rte_swx_match_field_params *field = &params->fields[i];
8199                 struct match_field *f = &t->fields[i];
8200
8201                 f->match_type = field->match_type;
8202                 f->field = header ?
8203                         header_field_parse(p, field->name, NULL) :
8204                         metadata_field_parse(p, field->name);
8205         }
8206         t->n_fields = params->n_fields;
8207         t->header = header;
8208
8209         for (i = 0; i < params->n_actions; i++)
8210                 t->actions[i] = action_find(p, params->action_names[i]);
8211         t->default_action = default_action;
8212         if (default_action->st)
8213                 memcpy(t->default_action_data,
8214                        params->default_action_data,
8215                        default_action->st->n_bits / 8);
8216         t->n_actions = params->n_actions;
8217         t->default_action_is_const = params->default_action_is_const;
8218         t->action_data_size_max = action_data_size_max;
8219
8220         t->size = size;
8221         t->id = p->n_tables;
8222
8223         /* Node add to tailq. */
8224         TAILQ_INSERT_TAIL(&p->tables, t, node);
8225         p->n_tables++;
8226
8227         return 0;
8228 }
8229
8230 static struct rte_swx_table_params *
8231 table_params_get(struct table *table)
8232 {
8233         struct rte_swx_table_params *params;
8234         struct field *first, *last;
8235         uint8_t *key_mask;
8236         uint32_t key_size, key_offset, action_data_size, i;
8237
8238         /* Memory allocation. */
8239         params = calloc(1, sizeof(struct rte_swx_table_params));
8240         if (!params)
8241                 return NULL;
8242
8243         /* Find first (smallest offset) and last (biggest offset) match fields. */
8244         first = table->fields[0].field;
8245         last = table->fields[0].field;
8246
8247         for (i = 0; i < table->n_fields; i++) {
8248                 struct field *f = table->fields[i].field;
8249
8250                 if (f->offset < first->offset)
8251                         first = f;
8252
8253                 if (f->offset > last->offset)
8254                         last = f;
8255         }
8256
8257         /* Key offset and size. */
8258         key_offset = first->offset / 8;
8259         key_size = (last->offset + last->n_bits - first->offset) / 8;
8260
8261         /* Memory allocation. */
8262         key_mask = calloc(1, key_size);
8263         if (!key_mask) {
8264                 free(params);
8265                 return NULL;
8266         }
8267
8268         /* Key mask. */
8269         for (i = 0; i < table->n_fields; i++) {
8270                 struct field *f = table->fields[i].field;
8271                 uint32_t start = (f->offset - first->offset) / 8;
8272                 size_t size = f->n_bits / 8;
8273
8274                 memset(&key_mask[start], 0xFF, size);
8275         }
8276
8277         /* Action data size. */
8278         action_data_size = 0;
8279         for (i = 0; i < table->n_actions; i++) {
8280                 struct action *action = table->actions[i];
8281                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8282
8283                 if (ads > action_data_size)
8284                         action_data_size = ads;
8285         }
8286
8287         /* Fill in. */
8288         params->match_type = table->type->match_type;
8289         params->key_size = key_size;
8290         params->key_offset = key_offset;
8291         params->key_mask0 = key_mask;
8292         params->action_data_size = action_data_size;
8293         params->n_keys_max = table->size;
8294
8295         return params;
8296 }
8297
8298 static void
8299 table_params_free(struct rte_swx_table_params *params)
8300 {
8301         if (!params)
8302                 return;
8303
8304         free(params->key_mask0);
8305         free(params);
8306 }
8307
8308 static int
8309 table_stub_lkp(void *table __rte_unused,
8310                void *mailbox __rte_unused,
8311                uint8_t **key __rte_unused,
8312                uint64_t *action_id __rte_unused,
8313                uint8_t **action_data __rte_unused,
8314                int *hit)
8315 {
8316         *hit = 0;
8317         return 1; /* DONE. */
8318 }
8319
8320 static int
8321 table_build(struct rte_swx_pipeline *p)
8322 {
8323         uint32_t i;
8324
8325         /* Per pipeline: table statistics. */
8326         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8327         CHECK(p->table_stats, ENOMEM);
8328
8329         for (i = 0; i < p->n_tables; i++) {
8330                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8331                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8332         }
8333
8334         /* Per thread: table runt-time. */
8335         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8336                 struct thread *t = &p->threads[i];
8337                 struct table *table;
8338
8339                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8340                 CHECK(t->tables, ENOMEM);
8341
8342                 TAILQ_FOREACH(table, &p->tables, node) {
8343                         struct table_runtime *r = &t->tables[table->id];
8344
8345                         if (table->type) {
8346                                 uint64_t size;
8347
8348                                 size = table->type->ops.mailbox_size_get();
8349
8350                                 /* r->func. */
8351                                 r->func = table->type->ops.lkp;
8352
8353                                 /* r->mailbox. */
8354                                 if (size) {
8355                                         r->mailbox = calloc(1, size);
8356                                         CHECK(r->mailbox, ENOMEM);
8357                                 }
8358
8359                                 /* r->key. */
8360                                 r->key = table->header ?
8361                                         &t->structs[table->header->struct_id] :
8362                                         &t->structs[p->metadata_struct_id];
8363                         } else {
8364                                 r->func = table_stub_lkp;
8365                         }
8366                 }
8367         }
8368
8369         return 0;
8370 }
8371
8372 static void
8373 table_build_free(struct rte_swx_pipeline *p)
8374 {
8375         uint32_t i;
8376
8377         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8378                 struct thread *t = &p->threads[i];
8379                 uint32_t j;
8380
8381                 if (!t->tables)
8382                         continue;
8383
8384                 for (j = 0; j < p->n_tables; j++) {
8385                         struct table_runtime *r = &t->tables[j];
8386
8387                         free(r->mailbox);
8388                 }
8389
8390                 free(t->tables);
8391                 t->tables = NULL;
8392         }
8393
8394         if (p->table_stats) {
8395                 for (i = 0; i < p->n_tables; i++)
8396                         free(p->table_stats[i].n_pkts_action);
8397
8398                 free(p->table_stats);
8399         }
8400 }
8401
8402 static void
8403 table_free(struct rte_swx_pipeline *p)
8404 {
8405         table_build_free(p);
8406
8407         /* Tables. */
8408         for ( ; ; ) {
8409                 struct table *elem;
8410
8411                 elem = TAILQ_FIRST(&p->tables);
8412                 if (!elem)
8413                         break;
8414
8415                 TAILQ_REMOVE(&p->tables, elem, node);
8416                 free(elem->fields);
8417                 free(elem->actions);
8418                 free(elem->default_action_data);
8419                 free(elem);
8420         }
8421
8422         /* Table types. */
8423         for ( ; ; ) {
8424                 struct table_type *elem;
8425
8426                 elem = TAILQ_FIRST(&p->table_types);
8427                 if (!elem)
8428                         break;
8429
8430                 TAILQ_REMOVE(&p->table_types, elem, node);
8431                 free(elem);
8432         }
8433 }
8434
8435 /*
8436  * Selector.
8437  */
8438 static struct selector *
8439 selector_find(struct rte_swx_pipeline *p, const char *name)
8440 {
8441         struct selector *s;
8442
8443         TAILQ_FOREACH(s, &p->selectors, node)
8444                 if (strcmp(s->name, name) == 0)
8445                         return s;
8446
8447         return NULL;
8448 }
8449
8450 static struct selector *
8451 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8452 {
8453         struct selector *s = NULL;
8454
8455         TAILQ_FOREACH(s, &p->selectors, node)
8456                 if (s->id == id)
8457                         return s;
8458
8459         return NULL;
8460 }
8461
8462 static int
8463 selector_fields_check(struct rte_swx_pipeline *p,
8464                       struct rte_swx_pipeline_selector_params *params,
8465                       struct header **header)
8466 {
8467         struct header *h0 = NULL;
8468         struct field *hf, *mf;
8469         uint32_t i;
8470
8471         /* Return if no selector fields. */
8472         if (!params->n_selector_fields || !params->selector_field_names)
8473                 return -EINVAL;
8474
8475         /* Check that all the selector fields either belong to the same header
8476          * or are all meta-data fields.
8477          */
8478         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8479         mf = metadata_field_parse(p, params->selector_field_names[0]);
8480         if (!hf && !mf)
8481                 return -EINVAL;
8482
8483         for (i = 1; i < params->n_selector_fields; i++)
8484                 if (h0) {
8485                         struct header *h;
8486
8487                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8488                         if (!hf || (h->id != h0->id))
8489                                 return -EINVAL;
8490                 } else {
8491                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8492                         if (!mf)
8493                                 return -EINVAL;
8494                 }
8495
8496         /* Check that there are no duplicated match fields. */
8497         for (i = 0; i < params->n_selector_fields; i++) {
8498                 const char *field_name = params->selector_field_names[i];
8499                 uint32_t j;
8500
8501                 for (j = i + 1; j < params->n_selector_fields; j++)
8502                         if (!strcmp(params->selector_field_names[j], field_name))
8503                                 return -EINVAL;
8504         }
8505
8506         /* Return. */
8507         if (header)
8508                 *header = h0;
8509
8510         return 0;
8511 }
8512
8513 int
8514 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8515                                  const char *name,
8516                                  struct rte_swx_pipeline_selector_params *params)
8517 {
8518         struct selector *s;
8519         struct header *selector_header = NULL;
8520         struct field *group_id_field, *member_id_field;
8521         uint32_t i;
8522         int status = 0;
8523
8524         CHECK(p, EINVAL);
8525
8526         CHECK_NAME(name, EINVAL);
8527         CHECK(!table_find(p, name), EEXIST);
8528         CHECK(!selector_find(p, name), EEXIST);
8529         CHECK(!learner_find(p, name), EEXIST);
8530
8531         CHECK(params, EINVAL);
8532
8533         CHECK_NAME(params->group_id_field_name, EINVAL);
8534         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8535         CHECK(group_id_field, EINVAL);
8536
8537         for (i = 0; i < params->n_selector_fields; i++) {
8538                 const char *field_name = params->selector_field_names[i];
8539
8540                 CHECK_NAME(field_name, EINVAL);
8541         }
8542         status = selector_fields_check(p, params, &selector_header);
8543         if (status)
8544                 return status;
8545
8546         CHECK_NAME(params->member_id_field_name, EINVAL);
8547         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8548         CHECK(member_id_field, EINVAL);
8549
8550         CHECK(params->n_groups_max, EINVAL);
8551
8552         CHECK(params->n_members_per_group_max, EINVAL);
8553
8554         /* Memory allocation. */
8555         s = calloc(1, sizeof(struct selector));
8556         if (!s) {
8557                 status = -ENOMEM;
8558                 goto error;
8559         }
8560
8561         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8562         if (!s->selector_fields) {
8563                 status = -ENOMEM;
8564                 goto error;
8565         }
8566
8567         /* Node initialization. */
8568         strcpy(s->name, name);
8569
8570         s->group_id_field = group_id_field;
8571
8572         for (i = 0; i < params->n_selector_fields; i++) {
8573                 const char *field_name = params->selector_field_names[i];
8574
8575                 s->selector_fields[i] = selector_header ?
8576                         header_field_parse(p, field_name, NULL) :
8577                         metadata_field_parse(p, field_name);
8578         }
8579
8580         s->n_selector_fields = params->n_selector_fields;
8581
8582         s->selector_header = selector_header;
8583
8584         s->member_id_field = member_id_field;
8585
8586         s->n_groups_max = params->n_groups_max;
8587
8588         s->n_members_per_group_max = params->n_members_per_group_max;
8589
8590         s->id = p->n_selectors;
8591
8592         /* Node add to tailq. */
8593         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8594         p->n_selectors++;
8595
8596         return 0;
8597
8598 error:
8599         if (!s)
8600                 return status;
8601
8602         free(s->selector_fields);
8603
8604         free(s);
8605
8606         return status;
8607 }
8608
8609 static void
8610 selector_params_free(struct rte_swx_table_selector_params *params)
8611 {
8612         if (!params)
8613                 return;
8614
8615         free(params->selector_mask);
8616
8617         free(params);
8618 }
8619
8620 static struct rte_swx_table_selector_params *
8621 selector_table_params_get(struct selector *s)
8622 {
8623         struct rte_swx_table_selector_params *params = NULL;
8624         struct field *first, *last;
8625         uint32_t i;
8626
8627         /* Memory allocation. */
8628         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8629         if (!params)
8630                 goto error;
8631
8632         /* Group ID. */
8633         params->group_id_offset = s->group_id_field->offset / 8;
8634
8635         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8636         first = s->selector_fields[0];
8637         last = s->selector_fields[0];
8638
8639         for (i = 0; i < s->n_selector_fields; i++) {
8640                 struct field *f = s->selector_fields[i];
8641
8642                 if (f->offset < first->offset)
8643                         first = f;
8644
8645                 if (f->offset > last->offset)
8646                         last = f;
8647         }
8648
8649         /* Selector offset and size. */
8650         params->selector_offset = first->offset / 8;
8651         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8652
8653         /* Memory allocation. */
8654         params->selector_mask = calloc(1, params->selector_size);
8655         if (!params->selector_mask)
8656                 goto error;
8657
8658         /* Selector mask. */
8659         for (i = 0; i < s->n_selector_fields; i++) {
8660                 struct field *f = s->selector_fields[i];
8661                 uint32_t start = (f->offset - first->offset) / 8;
8662                 size_t size = f->n_bits / 8;
8663
8664                 memset(&params->selector_mask[start], 0xFF, size);
8665         }
8666
8667         /* Member ID. */
8668         params->member_id_offset = s->member_id_field->offset / 8;
8669
8670         /* Maximum number of groups. */
8671         params->n_groups_max = s->n_groups_max;
8672
8673         /* Maximum number of members per group. */
8674         params->n_members_per_group_max = s->n_members_per_group_max;
8675
8676         return params;
8677
8678 error:
8679         selector_params_free(params);
8680         return NULL;
8681 }
8682
8683 static void
8684 selector_build_free(struct rte_swx_pipeline *p)
8685 {
8686         uint32_t i;
8687
8688         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8689                 struct thread *t = &p->threads[i];
8690                 uint32_t j;
8691
8692                 if (!t->selectors)
8693                         continue;
8694
8695                 for (j = 0; j < p->n_selectors; j++) {
8696                         struct selector_runtime *r = &t->selectors[j];
8697
8698                         free(r->mailbox);
8699                 }
8700
8701                 free(t->selectors);
8702                 t->selectors = NULL;
8703         }
8704
8705         free(p->selector_stats);
8706         p->selector_stats = NULL;
8707 }
8708
8709 static int
8710 selector_build(struct rte_swx_pipeline *p)
8711 {
8712         uint32_t i;
8713         int status = 0;
8714
8715         /* Per pipeline: selector statistics. */
8716         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8717         if (!p->selector_stats) {
8718                 status = -ENOMEM;
8719                 goto error;
8720         }
8721
8722         /* Per thread: selector run-time. */
8723         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8724                 struct thread *t = &p->threads[i];
8725                 struct selector *s;
8726
8727                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8728                 if (!t->selectors) {
8729                         status = -ENOMEM;
8730                         goto error;
8731                 }
8732
8733                 TAILQ_FOREACH(s, &p->selectors, node) {
8734                         struct selector_runtime *r = &t->selectors[s->id];
8735                         uint64_t size;
8736
8737                         /* r->mailbox. */
8738                         size = rte_swx_table_selector_mailbox_size_get();
8739                         if (size) {
8740                                 r->mailbox = calloc(1, size);
8741                                 if (!r->mailbox) {
8742                                         status = -ENOMEM;
8743                                         goto error;
8744                                 }
8745                         }
8746
8747                         /* r->group_id_buffer. */
8748                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8749
8750                         /* r->selector_buffer. */
8751                         r->selector_buffer = s->selector_header ?
8752                                 &t->structs[s->selector_header->struct_id] :
8753                                 &t->structs[p->metadata_struct_id];
8754
8755                         /* r->member_id_buffer. */
8756                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8757                 }
8758         }
8759
8760         return 0;
8761
8762 error:
8763         selector_build_free(p);
8764         return status;
8765 }
8766
8767 static void
8768 selector_free(struct rte_swx_pipeline *p)
8769 {
8770         selector_build_free(p);
8771
8772         /* Selector tables. */
8773         for ( ; ; ) {
8774                 struct selector *elem;
8775
8776                 elem = TAILQ_FIRST(&p->selectors);
8777                 if (!elem)
8778                         break;
8779
8780                 TAILQ_REMOVE(&p->selectors, elem, node);
8781                 free(elem->selector_fields);
8782                 free(elem);
8783         }
8784 }
8785
8786 /*
8787  * Learner table.
8788  */
8789 static struct learner *
8790 learner_find(struct rte_swx_pipeline *p, const char *name)
8791 {
8792         struct learner *l;
8793
8794         TAILQ_FOREACH(l, &p->learners, node)
8795                 if (!strcmp(l->name, name))
8796                         return l;
8797
8798         return NULL;
8799 }
8800
8801 static struct learner *
8802 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8803 {
8804         struct learner *l = NULL;
8805
8806         TAILQ_FOREACH(l, &p->learners, node)
8807                 if (l->id == id)
8808                         return l;
8809
8810         return NULL;
8811 }
8812
8813 static int
8814 learner_match_fields_check(struct rte_swx_pipeline *p,
8815                            struct rte_swx_pipeline_learner_params *params,
8816                            struct header **header)
8817 {
8818         struct header *h0 = NULL;
8819         struct field *hf, *mf;
8820         uint32_t i;
8821
8822         /* Return if no match fields. */
8823         if (!params->n_fields || !params->field_names)
8824                 return -EINVAL;
8825
8826         /* Check that all the match fields either belong to the same header
8827          * or are all meta-data fields.
8828          */
8829         hf = header_field_parse(p, params->field_names[0], &h0);
8830         mf = metadata_field_parse(p, params->field_names[0]);
8831         if (!hf && !mf)
8832                 return -EINVAL;
8833
8834         for (i = 1; i < params->n_fields; i++)
8835                 if (h0) {
8836                         struct header *h;
8837
8838                         hf = header_field_parse(p, params->field_names[i], &h);
8839                         if (!hf || (h->id != h0->id))
8840                                 return -EINVAL;
8841                 } else {
8842                         mf = metadata_field_parse(p, params->field_names[i]);
8843                         if (!mf)
8844                                 return -EINVAL;
8845                 }
8846
8847         /* Check that there are no duplicated match fields. */
8848         for (i = 0; i < params->n_fields; i++) {
8849                 const char *field_name = params->field_names[i];
8850                 uint32_t j;
8851
8852                 for (j = i + 1; j < params->n_fields; j++)
8853                         if (!strcmp(params->field_names[j], field_name))
8854                                 return -EINVAL;
8855         }
8856
8857         /* Return. */
8858         if (header)
8859                 *header = h0;
8860
8861         return 0;
8862 }
8863
8864 static int
8865 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8866 {
8867         struct struct_type *mst = p->metadata_st, *ast = a->st;
8868         struct field *mf, *af;
8869         uint32_t mf_pos, i;
8870
8871         if (!ast) {
8872                 if (mf_name)
8873                         return -EINVAL;
8874
8875                 return 0;
8876         }
8877
8878         /* Check that mf_name is the name of a valid meta-data field. */
8879         CHECK_NAME(mf_name, EINVAL);
8880         mf = metadata_field_parse(p, mf_name);
8881         CHECK(mf, EINVAL);
8882
8883         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8884          * all the action arguments.
8885          */
8886         mf_pos = mf - mst->fields;
8887         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8888
8889         /* Check that the size of each of the identified meta-data fields matches exactly the size
8890          * of the corresponding action argument.
8891          */
8892         for (i = 0; i < ast->n_fields; i++) {
8893                 mf = &mst->fields[mf_pos + i];
8894                 af = &ast->fields[i];
8895
8896                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8897         }
8898
8899         return 0;
8900 }
8901
8902 static int
8903 learner_action_learning_check(struct rte_swx_pipeline *p,
8904                               struct action *action,
8905                               const char **action_names,
8906                               uint32_t n_actions)
8907 {
8908         uint32_t i;
8909
8910         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8911          * the action passed as argument to the "learn" instruction) is also enabled for the
8912          * current learner table.
8913          */
8914         for (i = 0; i < action->n_instructions; i++) {
8915                 struct instruction *instr = &action->instructions[i];
8916                 uint32_t found = 0, j;
8917
8918                 if (instr->type != INSTR_LEARNER_LEARN)
8919                         continue;
8920
8921                 for (j = 0; j < n_actions; j++) {
8922                         struct action *a;
8923
8924                         a = action_find(p, action_names[j]);
8925                         if (!a)
8926                                 return -EINVAL;
8927
8928                         if (a->id == instr->learn.action_id)
8929                                 found = 1;
8930                 }
8931
8932                 if (!found)
8933                         return -EINVAL;
8934         }
8935
8936         return 0;
8937 }
8938
8939 int
8940 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8941                               const char *name,
8942                               struct rte_swx_pipeline_learner_params *params,
8943                               uint32_t size,
8944                               uint32_t timeout)
8945 {
8946         struct learner *l = NULL;
8947         struct action *default_action;
8948         struct header *header = NULL;
8949         uint32_t action_data_size_max = 0, i;
8950         int status = 0;
8951
8952         CHECK(p, EINVAL);
8953
8954         CHECK_NAME(name, EINVAL);
8955         CHECK(!table_find(p, name), EEXIST);
8956         CHECK(!selector_find(p, name), EEXIST);
8957         CHECK(!learner_find(p, name), EEXIST);
8958
8959         CHECK(params, EINVAL);
8960
8961         /* Match checks. */
8962         status = learner_match_fields_check(p, params, &header);
8963         if (status)
8964                 return status;
8965
8966         /* Action checks. */
8967         CHECK(params->n_actions, EINVAL);
8968
8969         CHECK(params->action_names, EINVAL);
8970         for (i = 0; i < params->n_actions; i++) {
8971                 const char *action_name = params->action_names[i];
8972                 const char *action_field_name = params->action_field_names[i];
8973                 struct action *a;
8974                 uint32_t action_data_size;
8975
8976                 CHECK_NAME(action_name, EINVAL);
8977
8978                 a = action_find(p, action_name);
8979                 CHECK(a, EINVAL);
8980
8981                 status = learner_action_args_check(p, a, action_field_name);
8982                 if (status)
8983                         return status;
8984
8985                 status = learner_action_learning_check(p,
8986                                                        a,
8987                                                        params->action_names,
8988                                                        params->n_actions);
8989                 if (status)
8990                         return status;
8991
8992                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8993                 if (action_data_size > action_data_size_max)
8994                         action_data_size_max = action_data_size;
8995         }
8996
8997         CHECK_NAME(params->default_action_name, EINVAL);
8998         for (i = 0; i < p->n_actions; i++)
8999                 if (!strcmp(params->action_names[i],
9000                             params->default_action_name))
9001                         break;
9002         CHECK(i < params->n_actions, EINVAL);
9003
9004         default_action = action_find(p, params->default_action_name);
9005         CHECK((default_action->st && params->default_action_data) ||
9006               !params->default_action_data, EINVAL);
9007
9008         /* Any other checks. */
9009         CHECK(size, EINVAL);
9010         CHECK(timeout, EINVAL);
9011
9012         /* Memory allocation. */
9013         l = calloc(1, sizeof(struct learner));
9014         if (!l)
9015                 goto nomem;
9016
9017         l->fields = calloc(params->n_fields, sizeof(struct field *));
9018         if (!l->fields)
9019                 goto nomem;
9020
9021         l->actions = calloc(params->n_actions, sizeof(struct action *));
9022         if (!l->actions)
9023                 goto nomem;
9024
9025         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9026         if (!l->action_arg)
9027                 goto nomem;
9028
9029         if (action_data_size_max) {
9030                 l->default_action_data = calloc(1, action_data_size_max);
9031                 if (!l->default_action_data)
9032                         goto nomem;
9033         }
9034
9035         /* Node initialization. */
9036         strcpy(l->name, name);
9037
9038         for (i = 0; i < params->n_fields; i++) {
9039                 const char *field_name = params->field_names[i];
9040
9041                 l->fields[i] = header ?
9042                         header_field_parse(p, field_name, NULL) :
9043                         metadata_field_parse(p, field_name);
9044         }
9045
9046         l->n_fields = params->n_fields;
9047
9048         l->header = header;
9049
9050         for (i = 0; i < params->n_actions; i++) {
9051                 const char *mf_name = params->action_field_names[i];
9052
9053                 l->actions[i] = action_find(p, params->action_names[i]);
9054
9055                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9056         }
9057
9058         l->default_action = default_action;
9059
9060         if (default_action->st)
9061                 memcpy(l->default_action_data,
9062                        params->default_action_data,
9063                        default_action->st->n_bits / 8);
9064
9065         l->n_actions = params->n_actions;
9066
9067         l->default_action_is_const = params->default_action_is_const;
9068
9069         l->action_data_size_max = action_data_size_max;
9070
9071         l->size = size;
9072
9073         l->timeout = timeout;
9074
9075         l->id = p->n_learners;
9076
9077         /* Node add to tailq. */
9078         TAILQ_INSERT_TAIL(&p->learners, l, node);
9079         p->n_learners++;
9080
9081         return 0;
9082
9083 nomem:
9084         if (!l)
9085                 return -ENOMEM;
9086
9087         free(l->action_arg);
9088         free(l->actions);
9089         free(l->fields);
9090         free(l);
9091
9092         return -ENOMEM;
9093 }
9094
9095 static void
9096 learner_params_free(struct rte_swx_table_learner_params *params)
9097 {
9098         if (!params)
9099                 return;
9100
9101         free(params->key_mask0);
9102
9103         free(params);
9104 }
9105
9106 static struct rte_swx_table_learner_params *
9107 learner_params_get(struct learner *l)
9108 {
9109         struct rte_swx_table_learner_params *params = NULL;
9110         struct field *first, *last;
9111         uint32_t i;
9112
9113         /* Memory allocation. */
9114         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9115         if (!params)
9116                 goto error;
9117
9118         /* Find first (smallest offset) and last (biggest offset) match fields. */
9119         first = l->fields[0];
9120         last = l->fields[0];
9121
9122         for (i = 0; i < l->n_fields; i++) {
9123                 struct field *f = l->fields[i];
9124
9125                 if (f->offset < first->offset)
9126                         first = f;
9127
9128                 if (f->offset > last->offset)
9129                         last = f;
9130         }
9131
9132         /* Key offset and size. */
9133         params->key_offset = first->offset / 8;
9134         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9135
9136         /* Memory allocation. */
9137         params->key_mask0 = calloc(1, params->key_size);
9138         if (!params->key_mask0)
9139                 goto error;
9140
9141         /* Key mask. */
9142         for (i = 0; i < l->n_fields; i++) {
9143                 struct field *f = l->fields[i];
9144                 uint32_t start = (f->offset - first->offset) / 8;
9145                 size_t size = f->n_bits / 8;
9146
9147                 memset(&params->key_mask0[start], 0xFF, size);
9148         }
9149
9150         /* Action data size. */
9151         params->action_data_size = l->action_data_size_max;
9152
9153         /* Maximum number of keys. */
9154         params->n_keys_max = l->size;
9155
9156         /* Timeout. */
9157         params->key_timeout = l->timeout;
9158
9159         return params;
9160
9161 error:
9162         learner_params_free(params);
9163         return NULL;
9164 }
9165
9166 static void
9167 learner_build_free(struct rte_swx_pipeline *p)
9168 {
9169         uint32_t i;
9170
9171         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9172                 struct thread *t = &p->threads[i];
9173                 uint32_t j;
9174
9175                 if (!t->learners)
9176                         continue;
9177
9178                 for (j = 0; j < p->n_learners; j++) {
9179                         struct learner_runtime *r = &t->learners[j];
9180
9181                         free(r->mailbox);
9182                         free(r->action_data);
9183                 }
9184
9185                 free(t->learners);
9186                 t->learners = NULL;
9187         }
9188
9189         if (p->learner_stats) {
9190                 for (i = 0; i < p->n_learners; i++)
9191                         free(p->learner_stats[i].n_pkts_action);
9192
9193                 free(p->learner_stats);
9194         }
9195 }
9196
9197 static int
9198 learner_build(struct rte_swx_pipeline *p)
9199 {
9200         uint32_t i;
9201         int status = 0;
9202
9203         /* Per pipeline: learner statistics. */
9204         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9205         CHECK(p->learner_stats, ENOMEM);
9206
9207         for (i = 0; i < p->n_learners; i++) {
9208                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9209                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9210         }
9211
9212         /* Per thread: learner run-time. */
9213         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9214                 struct thread *t = &p->threads[i];
9215                 struct learner *l;
9216
9217                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9218                 if (!t->learners) {
9219                         status = -ENOMEM;
9220                         goto error;
9221                 }
9222
9223                 TAILQ_FOREACH(l, &p->learners, node) {
9224                         struct learner_runtime *r = &t->learners[l->id];
9225                         uint64_t size;
9226                         uint32_t j;
9227
9228                         /* r->mailbox. */
9229                         size = rte_swx_table_learner_mailbox_size_get();
9230                         if (size) {
9231                                 r->mailbox = calloc(1, size);
9232                                 if (!r->mailbox) {
9233                                         status = -ENOMEM;
9234                                         goto error;
9235                                 }
9236                         }
9237
9238                         /* r->key. */
9239                         r->key = l->header ?
9240                                 &t->structs[l->header->struct_id] :
9241                                 &t->structs[p->metadata_struct_id];
9242
9243                         /* r->action_data. */
9244                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9245                         if (!r->action_data) {
9246                                 status = -ENOMEM;
9247                                 goto error;
9248                         }
9249
9250                         for (j = 0; j < l->n_actions; j++) {
9251                                 struct action *a = l->actions[j];
9252                                 struct field *mf = l->action_arg[j];
9253                                 uint8_t *m = t->structs[p->metadata_struct_id];
9254
9255                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9256                         }
9257                 }
9258         }
9259
9260         return 0;
9261
9262 error:
9263         learner_build_free(p);
9264         return status;
9265 }
9266
9267 static void
9268 learner_free(struct rte_swx_pipeline *p)
9269 {
9270         learner_build_free(p);
9271
9272         /* Learner tables. */
9273         for ( ; ; ) {
9274                 struct learner *l;
9275
9276                 l = TAILQ_FIRST(&p->learners);
9277                 if (!l)
9278                         break;
9279
9280                 TAILQ_REMOVE(&p->learners, l, node);
9281                 free(l->fields);
9282                 free(l->actions);
9283                 free(l->action_arg);
9284                 free(l->default_action_data);
9285                 free(l);
9286         }
9287 }
9288
9289 /*
9290  * Table state.
9291  */
9292 static int
9293 table_state_build(struct rte_swx_pipeline *p)
9294 {
9295         struct table *table;
9296         struct selector *s;
9297         struct learner *l;
9298
9299         p->table_state = calloc(p->n_tables + p->n_selectors,
9300                                 sizeof(struct rte_swx_table_state));
9301         CHECK(p->table_state, ENOMEM);
9302
9303         TAILQ_FOREACH(table, &p->tables, node) {
9304                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9305
9306                 if (table->type) {
9307                         struct rte_swx_table_params *params;
9308
9309                         /* ts->obj. */
9310                         params = table_params_get(table);
9311                         CHECK(params, ENOMEM);
9312
9313                         ts->obj = table->type->ops.create(params,
9314                                 NULL,
9315                                 table->args,
9316                                 p->numa_node);
9317
9318                         table_params_free(params);
9319                         CHECK(ts->obj, ENODEV);
9320                 }
9321
9322                 /* ts->default_action_data. */
9323                 if (table->action_data_size_max) {
9324                         ts->default_action_data =
9325                                 malloc(table->action_data_size_max);
9326                         CHECK(ts->default_action_data, ENOMEM);
9327
9328                         memcpy(ts->default_action_data,
9329                                table->default_action_data,
9330                                table->action_data_size_max);
9331                 }
9332
9333                 /* ts->default_action_id. */
9334                 ts->default_action_id = table->default_action->id;
9335         }
9336
9337         TAILQ_FOREACH(s, &p->selectors, node) {
9338                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9339                 struct rte_swx_table_selector_params *params;
9340
9341                 /* ts->obj. */
9342                 params = selector_table_params_get(s);
9343                 CHECK(params, ENOMEM);
9344
9345                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9346
9347                 selector_params_free(params);
9348                 CHECK(ts->obj, ENODEV);
9349         }
9350
9351         TAILQ_FOREACH(l, &p->learners, node) {
9352                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9353                         p->n_selectors + l->id];
9354                 struct rte_swx_table_learner_params *params;
9355
9356                 /* ts->obj. */
9357                 params = learner_params_get(l);
9358                 CHECK(params, ENOMEM);
9359
9360                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9361                 learner_params_free(params);
9362                 CHECK(ts->obj, ENODEV);
9363
9364                 /* ts->default_action_data. */
9365                 if (l->action_data_size_max) {
9366                         ts->default_action_data = malloc(l->action_data_size_max);
9367                         CHECK(ts->default_action_data, ENOMEM);
9368
9369                         memcpy(ts->default_action_data,
9370                                l->default_action_data,
9371                                l->action_data_size_max);
9372                 }
9373
9374                 /* ts->default_action_id. */
9375                 ts->default_action_id = l->default_action->id;
9376         }
9377
9378         return 0;
9379 }
9380
9381 static void
9382 table_state_build_free(struct rte_swx_pipeline *p)
9383 {
9384         uint32_t i;
9385
9386         if (!p->table_state)
9387                 return;
9388
9389         for (i = 0; i < p->n_tables; i++) {
9390                 struct rte_swx_table_state *ts = &p->table_state[i];
9391                 struct table *table = table_find_by_id(p, i);
9392
9393                 /* ts->obj. */
9394                 if (table->type && ts->obj)
9395                         table->type->ops.free(ts->obj);
9396
9397                 /* ts->default_action_data. */
9398                 free(ts->default_action_data);
9399         }
9400
9401         for (i = 0; i < p->n_selectors; i++) {
9402                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9403
9404                 /* ts->obj. */
9405                 if (ts->obj)
9406                         rte_swx_table_selector_free(ts->obj);
9407         }
9408
9409         for (i = 0; i < p->n_learners; i++) {
9410                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9411
9412                 /* ts->obj. */
9413                 if (ts->obj)
9414                         rte_swx_table_learner_free(ts->obj);
9415
9416                 /* ts->default_action_data. */
9417                 free(ts->default_action_data);
9418         }
9419
9420         free(p->table_state);
9421         p->table_state = NULL;
9422 }
9423
9424 static void
9425 table_state_free(struct rte_swx_pipeline *p)
9426 {
9427         table_state_build_free(p);
9428 }
9429
9430 /*
9431  * Register array.
9432  */
9433 static struct regarray *
9434 regarray_find(struct rte_swx_pipeline *p, const char *name)
9435 {
9436         struct regarray *elem;
9437
9438         TAILQ_FOREACH(elem, &p->regarrays, node)
9439                 if (!strcmp(elem->name, name))
9440                         return elem;
9441
9442         return NULL;
9443 }
9444
9445 static struct regarray *
9446 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9447 {
9448         struct regarray *elem = NULL;
9449
9450         TAILQ_FOREACH(elem, &p->regarrays, node)
9451                 if (elem->id == id)
9452                         return elem;
9453
9454         return NULL;
9455 }
9456
9457 int
9458 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9459                               const char *name,
9460                               uint32_t size,
9461                               uint64_t init_val)
9462 {
9463         struct regarray *r;
9464
9465         CHECK(p, EINVAL);
9466
9467         CHECK_NAME(name, EINVAL);
9468         CHECK(!regarray_find(p, name), EEXIST);
9469
9470         CHECK(size, EINVAL);
9471         size = rte_align32pow2(size);
9472
9473         /* Memory allocation. */
9474         r = calloc(1, sizeof(struct regarray));
9475         CHECK(r, ENOMEM);
9476
9477         /* Node initialization. */
9478         strcpy(r->name, name);
9479         r->init_val = init_val;
9480         r->size = size;
9481         r->id = p->n_regarrays;
9482
9483         /* Node add to tailq. */
9484         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9485         p->n_regarrays++;
9486
9487         return 0;
9488 }
9489
9490 static int
9491 regarray_build(struct rte_swx_pipeline *p)
9492 {
9493         struct regarray *regarray;
9494
9495         if (!p->n_regarrays)
9496                 return 0;
9497
9498         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9499         CHECK(p->regarray_runtime, ENOMEM);
9500
9501         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9502                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9503                 uint32_t i;
9504
9505                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9506                                          RTE_CACHE_LINE_SIZE,
9507                                          p->numa_node);
9508                 CHECK(r->regarray, ENOMEM);
9509
9510                 if (regarray->init_val)
9511                         for (i = 0; i < regarray->size; i++)
9512                                 r->regarray[i] = regarray->init_val;
9513
9514                 r->size_mask = regarray->size - 1;
9515         }
9516
9517         return 0;
9518 }
9519
9520 static void
9521 regarray_build_free(struct rte_swx_pipeline *p)
9522 {
9523         uint32_t i;
9524
9525         if (!p->regarray_runtime)
9526                 return;
9527
9528         for (i = 0; i < p->n_regarrays; i++) {
9529                 struct regarray *regarray = regarray_find_by_id(p, i);
9530                 struct regarray_runtime *r = &p->regarray_runtime[i];
9531
9532                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9533         }
9534
9535         free(p->regarray_runtime);
9536         p->regarray_runtime = NULL;
9537 }
9538
9539 static void
9540 regarray_free(struct rte_swx_pipeline *p)
9541 {
9542         regarray_build_free(p);
9543
9544         for ( ; ; ) {
9545                 struct regarray *elem;
9546
9547                 elem = TAILQ_FIRST(&p->regarrays);
9548                 if (!elem)
9549                         break;
9550
9551                 TAILQ_REMOVE(&p->regarrays, elem, node);
9552                 free(elem);
9553         }
9554 }
9555
9556 /*
9557  * Meter array.
9558  */
9559 static struct meter_profile *
9560 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9561 {
9562         struct meter_profile *elem;
9563
9564         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9565                 if (!strcmp(elem->name, name))
9566                         return elem;
9567
9568         return NULL;
9569 }
9570
9571 static struct metarray *
9572 metarray_find(struct rte_swx_pipeline *p, const char *name)
9573 {
9574         struct metarray *elem;
9575
9576         TAILQ_FOREACH(elem, &p->metarrays, node)
9577                 if (!strcmp(elem->name, name))
9578                         return elem;
9579
9580         return NULL;
9581 }
9582
9583 static struct metarray *
9584 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9585 {
9586         struct metarray *elem = NULL;
9587
9588         TAILQ_FOREACH(elem, &p->metarrays, node)
9589                 if (elem->id == id)
9590                         return elem;
9591
9592         return NULL;
9593 }
9594
9595 int
9596 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9597                                  const char *name,
9598                                  uint32_t size)
9599 {
9600         struct metarray *m;
9601
9602         CHECK(p, EINVAL);
9603
9604         CHECK_NAME(name, EINVAL);
9605         CHECK(!metarray_find(p, name), EEXIST);
9606
9607         CHECK(size, EINVAL);
9608         size = rte_align32pow2(size);
9609
9610         /* Memory allocation. */
9611         m = calloc(1, sizeof(struct metarray));
9612         CHECK(m, ENOMEM);
9613
9614         /* Node initialization. */
9615         strcpy(m->name, name);
9616         m->size = size;
9617         m->id = p->n_metarrays;
9618
9619         /* Node add to tailq. */
9620         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9621         p->n_metarrays++;
9622
9623         return 0;
9624 }
9625
9626 struct meter_profile meter_profile_default = {
9627         .node = {0},
9628         .name = "",
9629         .params = {0},
9630
9631         .profile = {
9632                 .cbs = 10000,
9633                 .pbs = 10000,
9634                 .cir_period = 1,
9635                 .cir_bytes_per_period = 1,
9636                 .pir_period = 1,
9637                 .pir_bytes_per_period = 1,
9638         },
9639
9640         .n_users = 0,
9641 };
9642
9643 static void
9644 meter_init(struct meter *m)
9645 {
9646         memset(m, 0, sizeof(struct meter));
9647         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9648         m->profile = &meter_profile_default;
9649         m->color_mask = RTE_COLOR_GREEN;
9650
9651         meter_profile_default.n_users++;
9652 }
9653
9654 static int
9655 metarray_build(struct rte_swx_pipeline *p)
9656 {
9657         struct metarray *m;
9658
9659         if (!p->n_metarrays)
9660                 return 0;
9661
9662         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9663         CHECK(p->metarray_runtime, ENOMEM);
9664
9665         TAILQ_FOREACH(m, &p->metarrays, node) {
9666                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9667                 uint32_t i;
9668
9669                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9670                                          RTE_CACHE_LINE_SIZE,
9671                                          p->numa_node);
9672                 CHECK(r->metarray, ENOMEM);
9673
9674                 for (i = 0; i < m->size; i++)
9675                         meter_init(&r->metarray[i]);
9676
9677                 r->size_mask = m->size - 1;
9678         }
9679
9680         return 0;
9681 }
9682
9683 static void
9684 metarray_build_free(struct rte_swx_pipeline *p)
9685 {
9686         uint32_t i;
9687
9688         if (!p->metarray_runtime)
9689                 return;
9690
9691         for (i = 0; i < p->n_metarrays; i++) {
9692                 struct metarray *m = metarray_find_by_id(p, i);
9693                 struct metarray_runtime *r = &p->metarray_runtime[i];
9694
9695                 env_free(r->metarray, m->size * sizeof(struct meter));
9696         }
9697
9698         free(p->metarray_runtime);
9699         p->metarray_runtime = NULL;
9700 }
9701
9702 static void
9703 metarray_free(struct rte_swx_pipeline *p)
9704 {
9705         metarray_build_free(p);
9706
9707         /* Meter arrays. */
9708         for ( ; ; ) {
9709                 struct metarray *elem;
9710
9711                 elem = TAILQ_FIRST(&p->metarrays);
9712                 if (!elem)
9713                         break;
9714
9715                 TAILQ_REMOVE(&p->metarrays, elem, node);
9716                 free(elem);
9717         }
9718
9719         /* Meter profiles. */
9720         for ( ; ; ) {
9721                 struct meter_profile *elem;
9722
9723                 elem = TAILQ_FIRST(&p->meter_profiles);
9724                 if (!elem)
9725                         break;
9726
9727                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9728                 free(elem);
9729         }
9730 }
9731
9732 /*
9733  * Pipeline.
9734  */
9735 int
9736 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9737 {
9738         struct rte_swx_pipeline *pipeline;
9739
9740         /* Check input parameters. */
9741         CHECK(p, EINVAL);
9742
9743         /* Memory allocation. */
9744         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9745         CHECK(pipeline, ENOMEM);
9746
9747         /* Initialization. */
9748         TAILQ_INIT(&pipeline->struct_types);
9749         TAILQ_INIT(&pipeline->port_in_types);
9750         TAILQ_INIT(&pipeline->ports_in);
9751         TAILQ_INIT(&pipeline->port_out_types);
9752         TAILQ_INIT(&pipeline->ports_out);
9753         TAILQ_INIT(&pipeline->extern_types);
9754         TAILQ_INIT(&pipeline->extern_objs);
9755         TAILQ_INIT(&pipeline->extern_funcs);
9756         TAILQ_INIT(&pipeline->headers);
9757         TAILQ_INIT(&pipeline->actions);
9758         TAILQ_INIT(&pipeline->table_types);
9759         TAILQ_INIT(&pipeline->tables);
9760         TAILQ_INIT(&pipeline->selectors);
9761         TAILQ_INIT(&pipeline->learners);
9762         TAILQ_INIT(&pipeline->regarrays);
9763         TAILQ_INIT(&pipeline->meter_profiles);
9764         TAILQ_INIT(&pipeline->metarrays);
9765
9766         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9767         pipeline->numa_node = numa_node;
9768
9769         *p = pipeline;
9770         return 0;
9771 }
9772
9773 void
9774 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9775 {
9776         if (!p)
9777                 return;
9778
9779         free(p->instructions);
9780
9781         metarray_free(p);
9782         regarray_free(p);
9783         table_state_free(p);
9784         learner_free(p);
9785         selector_free(p);
9786         table_free(p);
9787         action_free(p);
9788         metadata_free(p);
9789         header_free(p);
9790         extern_func_free(p);
9791         extern_obj_free(p);
9792         port_out_free(p);
9793         port_in_free(p);
9794         struct_free(p);
9795
9796         free(p);
9797 }
9798
9799 int
9800 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9801                                      const char **instructions,
9802                                      uint32_t n_instructions)
9803 {
9804         int err;
9805         uint32_t i;
9806
9807         err = instruction_config(p, NULL, instructions, n_instructions);
9808         if (err)
9809                 return err;
9810
9811         /* Thread instruction pointer reset. */
9812         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9813                 struct thread *t = &p->threads[i];
9814
9815                 thread_ip_reset(p, t);
9816         }
9817
9818         return 0;
9819 }
9820
9821 int
9822 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9823 {
9824         int status;
9825
9826         CHECK(p, EINVAL);
9827         CHECK(p->build_done == 0, EEXIST);
9828
9829         status = port_in_build(p);
9830         if (status)
9831                 goto error;
9832
9833         status = port_out_build(p);
9834         if (status)
9835                 goto error;
9836
9837         status = struct_build(p);
9838         if (status)
9839                 goto error;
9840
9841         status = extern_obj_build(p);
9842         if (status)
9843                 goto error;
9844
9845         status = extern_func_build(p);
9846         if (status)
9847                 goto error;
9848
9849         status = header_build(p);
9850         if (status)
9851                 goto error;
9852
9853         status = metadata_build(p);
9854         if (status)
9855                 goto error;
9856
9857         status = action_build(p);
9858         if (status)
9859                 goto error;
9860
9861         status = table_build(p);
9862         if (status)
9863                 goto error;
9864
9865         status = selector_build(p);
9866         if (status)
9867                 goto error;
9868
9869         status = learner_build(p);
9870         if (status)
9871                 goto error;
9872
9873         status = table_state_build(p);
9874         if (status)
9875                 goto error;
9876
9877         status = regarray_build(p);
9878         if (status)
9879                 goto error;
9880
9881         status = metarray_build(p);
9882         if (status)
9883                 goto error;
9884
9885         p->build_done = 1;
9886         return 0;
9887
9888 error:
9889         metarray_build_free(p);
9890         regarray_build_free(p);
9891         table_state_build_free(p);
9892         learner_build_free(p);
9893         selector_build_free(p);
9894         table_build_free(p);
9895         action_build_free(p);
9896         metadata_build_free(p);
9897         header_build_free(p);
9898         extern_func_build_free(p);
9899         extern_obj_build_free(p);
9900         port_out_build_free(p);
9901         port_in_build_free(p);
9902         struct_build_free(p);
9903
9904         return status;
9905 }
9906
9907 void
9908 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9909 {
9910         uint32_t i;
9911
9912         for (i = 0; i < n_instructions; i++)
9913                 instr_exec(p);
9914 }
9915
9916 void
9917 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9918 {
9919         uint32_t i;
9920
9921         for (i = 0; i < p->n_ports_out; i++) {
9922                 struct port_out_runtime *port = &p->out[i];
9923
9924                 if (port->flush)
9925                         port->flush(port->obj);
9926         }
9927 }
9928
9929 /*
9930  * Control.
9931  */
9932 int
9933 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9934                               struct rte_swx_ctl_pipeline_info *pipeline)
9935 {
9936         struct action *action;
9937         struct table *table;
9938         uint32_t n_actions = 0, n_tables = 0;
9939
9940         if (!p || !pipeline)
9941                 return -EINVAL;
9942
9943         TAILQ_FOREACH(action, &p->actions, node)
9944                 n_actions++;
9945
9946         TAILQ_FOREACH(table, &p->tables, node)
9947                 n_tables++;
9948
9949         pipeline->n_ports_in = p->n_ports_in;
9950         pipeline->n_ports_out = p->n_ports_out;
9951         pipeline->n_actions = n_actions;
9952         pipeline->n_tables = n_tables;
9953         pipeline->n_selectors = p->n_selectors;
9954         pipeline->n_learners = p->n_learners;
9955         pipeline->n_regarrays = p->n_regarrays;
9956         pipeline->n_metarrays = p->n_metarrays;
9957
9958         return 0;
9959 }
9960
9961 int
9962 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9963 {
9964         if (!p || !numa_node)
9965                 return -EINVAL;
9966
9967         *numa_node = p->numa_node;
9968         return 0;
9969 }
9970
9971 int
9972 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9973                             uint32_t action_id,
9974                             struct rte_swx_ctl_action_info *action)
9975 {
9976         struct action *a = NULL;
9977
9978         if (!p || (action_id >= p->n_actions) || !action)
9979                 return -EINVAL;
9980
9981         a = action_find_by_id(p, action_id);
9982         if (!a)
9983                 return -EINVAL;
9984
9985         strcpy(action->name, a->name);
9986         action->n_args = a->st ? a->st->n_fields : 0;
9987         return 0;
9988 }
9989
9990 int
9991 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9992                                 uint32_t action_id,
9993                                 uint32_t action_arg_id,
9994                                 struct rte_swx_ctl_action_arg_info *action_arg)
9995 {
9996         struct action *a = NULL;
9997         struct field *arg = NULL;
9998
9999         if (!p || (action_id >= p->n_actions) || !action_arg)
10000                 return -EINVAL;
10001
10002         a = action_find_by_id(p, action_id);
10003         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10004                 return -EINVAL;
10005
10006         arg = &a->st->fields[action_arg_id];
10007         strcpy(action_arg->name, arg->name);
10008         action_arg->n_bits = arg->n_bits;
10009         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10010
10011         return 0;
10012 }
10013
10014 int
10015 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10016                            uint32_t table_id,
10017                            struct rte_swx_ctl_table_info *table)
10018 {
10019         struct table *t = NULL;
10020
10021         if (!p || !table)
10022                 return -EINVAL;
10023
10024         t = table_find_by_id(p, table_id);
10025         if (!t)
10026                 return -EINVAL;
10027
10028         strcpy(table->name, t->name);
10029         strcpy(table->args, t->args);
10030         table->n_match_fields = t->n_fields;
10031         table->n_actions = t->n_actions;
10032         table->default_action_is_const = t->default_action_is_const;
10033         table->size = t->size;
10034         return 0;
10035 }
10036
10037 int
10038 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10039         uint32_t table_id,
10040         uint32_t match_field_id,
10041         struct rte_swx_ctl_table_match_field_info *match_field)
10042 {
10043         struct table *t;
10044         struct match_field *f;
10045
10046         if (!p || (table_id >= p->n_tables) || !match_field)
10047                 return -EINVAL;
10048
10049         t = table_find_by_id(p, table_id);
10050         if (!t || (match_field_id >= t->n_fields))
10051                 return -EINVAL;
10052
10053         f = &t->fields[match_field_id];
10054         match_field->match_type = f->match_type;
10055         match_field->is_header = t->header ? 1 : 0;
10056         match_field->n_bits = f->field->n_bits;
10057         match_field->offset = f->field->offset;
10058
10059         return 0;
10060 }
10061
10062 int
10063 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10064         uint32_t table_id,
10065         uint32_t table_action_id,
10066         struct rte_swx_ctl_table_action_info *table_action)
10067 {
10068         struct table *t;
10069
10070         if (!p || (table_id >= p->n_tables) || !table_action)
10071                 return -EINVAL;
10072
10073         t = table_find_by_id(p, table_id);
10074         if (!t || (table_action_id >= t->n_actions))
10075                 return -EINVAL;
10076
10077         table_action->action_id = t->actions[table_action_id]->id;
10078
10079         return 0;
10080 }
10081
10082 int
10083 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10084                           uint32_t table_id,
10085                           struct rte_swx_table_ops *table_ops,
10086                           int *is_stub)
10087 {
10088         struct table *t;
10089
10090         if (!p || (table_id >= p->n_tables))
10091                 return -EINVAL;
10092
10093         t = table_find_by_id(p, table_id);
10094         if (!t)
10095                 return -EINVAL;
10096
10097         if (t->type) {
10098                 if (table_ops)
10099                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10100                 *is_stub = 0;
10101         } else {
10102                 *is_stub = 1;
10103         }
10104
10105         return 0;
10106 }
10107
10108 int
10109 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10110                               uint32_t selector_id,
10111                               struct rte_swx_ctl_selector_info *selector)
10112 {
10113         struct selector *s = NULL;
10114
10115         if (!p || !selector)
10116                 return -EINVAL;
10117
10118         s = selector_find_by_id(p, selector_id);
10119         if (!s)
10120                 return -EINVAL;
10121
10122         strcpy(selector->name, s->name);
10123
10124         selector->n_selector_fields = s->n_selector_fields;
10125         selector->n_groups_max = s->n_groups_max;
10126         selector->n_members_per_group_max = s->n_members_per_group_max;
10127
10128         return 0;
10129 }
10130
10131 int
10132 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10133          uint32_t selector_id,
10134          struct rte_swx_ctl_table_match_field_info *field)
10135 {
10136         struct selector *s;
10137
10138         if (!p || (selector_id >= p->n_selectors) || !field)
10139                 return -EINVAL;
10140
10141         s = selector_find_by_id(p, selector_id);
10142         if (!s)
10143                 return -EINVAL;
10144
10145         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10146         field->is_header = 0;
10147         field->n_bits = s->group_id_field->n_bits;
10148         field->offset = s->group_id_field->offset;
10149
10150         return 0;
10151 }
10152
10153 int
10154 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10155          uint32_t selector_id,
10156          uint32_t selector_field_id,
10157          struct rte_swx_ctl_table_match_field_info *field)
10158 {
10159         struct selector *s;
10160         struct field *f;
10161
10162         if (!p || (selector_id >= p->n_selectors) || !field)
10163                 return -EINVAL;
10164
10165         s = selector_find_by_id(p, selector_id);
10166         if (!s || (selector_field_id >= s->n_selector_fields))
10167                 return -EINVAL;
10168
10169         f = s->selector_fields[selector_field_id];
10170         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10171         field->is_header = s->selector_header ? 1 : 0;
10172         field->n_bits = f->n_bits;
10173         field->offset = f->offset;
10174
10175         return 0;
10176 }
10177
10178 int
10179 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10180          uint32_t selector_id,
10181          struct rte_swx_ctl_table_match_field_info *field)
10182 {
10183         struct selector *s;
10184
10185         if (!p || (selector_id >= p->n_selectors) || !field)
10186                 return -EINVAL;
10187
10188         s = selector_find_by_id(p, selector_id);
10189         if (!s)
10190                 return -EINVAL;
10191
10192         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10193         field->is_header = 0;
10194         field->n_bits = s->member_id_field->n_bits;
10195         field->offset = s->member_id_field->offset;
10196
10197         return 0;
10198 }
10199
10200 int
10201 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10202                              uint32_t learner_id,
10203                              struct rte_swx_ctl_learner_info *learner)
10204 {
10205         struct learner *l = NULL;
10206
10207         if (!p || !learner)
10208                 return -EINVAL;
10209
10210         l = learner_find_by_id(p, learner_id);
10211         if (!l)
10212                 return -EINVAL;
10213
10214         strcpy(learner->name, l->name);
10215
10216         learner->n_match_fields = l->n_fields;
10217         learner->n_actions = l->n_actions;
10218         learner->default_action_is_const = l->default_action_is_const;
10219         learner->size = l->size;
10220
10221         return 0;
10222 }
10223
10224 int
10225 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10226                                          uint32_t learner_id,
10227                                          uint32_t match_field_id,
10228                                          struct rte_swx_ctl_table_match_field_info *match_field)
10229 {
10230         struct learner *l;
10231         struct field *f;
10232
10233         if (!p || (learner_id >= p->n_learners) || !match_field)
10234                 return -EINVAL;
10235
10236         l = learner_find_by_id(p, learner_id);
10237         if (!l || (match_field_id >= l->n_fields))
10238                 return -EINVAL;
10239
10240         f = l->fields[match_field_id];
10241         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10242         match_field->is_header = l->header ? 1 : 0;
10243         match_field->n_bits = f->n_bits;
10244         match_field->offset = f->offset;
10245
10246         return 0;
10247 }
10248
10249 int
10250 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10251                                     uint32_t learner_id,
10252                                     uint32_t learner_action_id,
10253                                     struct rte_swx_ctl_table_action_info *learner_action)
10254 {
10255         struct learner *l;
10256
10257         if (!p || (learner_id >= p->n_learners) || !learner_action)
10258                 return -EINVAL;
10259
10260         l = learner_find_by_id(p, learner_id);
10261         if (!l || (learner_action_id >= l->n_actions))
10262                 return -EINVAL;
10263
10264         learner_action->action_id = l->actions[learner_action_id]->id;
10265
10266         return 0;
10267 }
10268
10269 int
10270 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10271                                  struct rte_swx_table_state **table_state)
10272 {
10273         if (!p || !table_state || !p->build_done)
10274                 return -EINVAL;
10275
10276         *table_state = p->table_state;
10277         return 0;
10278 }
10279
10280 int
10281 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10282                                  struct rte_swx_table_state *table_state)
10283 {
10284         if (!p || !table_state || !p->build_done)
10285                 return -EINVAL;
10286
10287         p->table_state = table_state;
10288         return 0;
10289 }
10290
10291 int
10292 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10293                                         uint32_t port_id,
10294                                         struct rte_swx_port_in_stats *stats)
10295 {
10296         struct port_in *port;
10297
10298         if (!p || !stats)
10299                 return -EINVAL;
10300
10301         port = port_in_find(p, port_id);
10302         if (!port)
10303                 return -EINVAL;
10304
10305         port->type->ops.stats_read(port->obj, stats);
10306         return 0;
10307 }
10308
10309 int
10310 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10311                                          uint32_t port_id,
10312                                          struct rte_swx_port_out_stats *stats)
10313 {
10314         struct port_out *port;
10315
10316         if (!p || !stats)
10317                 return -EINVAL;
10318
10319         port = port_out_find(p, port_id);
10320         if (!port)
10321                 return -EINVAL;
10322
10323         port->type->ops.stats_read(port->obj, stats);
10324         return 0;
10325 }
10326
10327 int
10328 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10329                                       const char *table_name,
10330                                       struct rte_swx_table_stats *stats)
10331 {
10332         struct table *table;
10333         struct table_statistics *table_stats;
10334
10335         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10336                 return -EINVAL;
10337
10338         table = table_find(p, table_name);
10339         if (!table)
10340                 return -EINVAL;
10341
10342         table_stats = &p->table_stats[table->id];
10343
10344         memcpy(stats->n_pkts_action,
10345                table_stats->n_pkts_action,
10346                p->n_actions * sizeof(uint64_t));
10347
10348         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10349         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10350
10351         return 0;
10352 }
10353
10354 int
10355 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10356         const char *selector_name,
10357         struct rte_swx_pipeline_selector_stats *stats)
10358 {
10359         struct selector *s;
10360
10361         if (!p || !selector_name || !selector_name[0] || !stats)
10362                 return -EINVAL;
10363
10364         s = selector_find(p, selector_name);
10365         if (!s)
10366                 return -EINVAL;
10367
10368         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10369
10370         return 0;
10371 }
10372
10373 int
10374 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10375                                         const char *learner_name,
10376                                         struct rte_swx_learner_stats *stats)
10377 {
10378         struct learner *l;
10379         struct learner_statistics *learner_stats;
10380
10381         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10382                 return -EINVAL;
10383
10384         l = learner_find(p, learner_name);
10385         if (!l)
10386                 return -EINVAL;
10387
10388         learner_stats = &p->learner_stats[l->id];
10389
10390         memcpy(stats->n_pkts_action,
10391                learner_stats->n_pkts_action,
10392                p->n_actions * sizeof(uint64_t));
10393
10394         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10395         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10396
10397         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10398         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10399
10400         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10401
10402         return 0;
10403 }
10404
10405 int
10406 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10407                               uint32_t regarray_id,
10408                               struct rte_swx_ctl_regarray_info *regarray)
10409 {
10410         struct regarray *r;
10411
10412         if (!p || !regarray)
10413                 return -EINVAL;
10414
10415         r = regarray_find_by_id(p, regarray_id);
10416         if (!r)
10417                 return -EINVAL;
10418
10419         strcpy(regarray->name, r->name);
10420         regarray->size = r->size;
10421         return 0;
10422 }
10423
10424 int
10425 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10426                                    const char *regarray_name,
10427                                    uint32_t regarray_index,
10428                                    uint64_t *value)
10429 {
10430         struct regarray *regarray;
10431         struct regarray_runtime *r;
10432
10433         if (!p || !regarray_name || !value)
10434                 return -EINVAL;
10435
10436         regarray = regarray_find(p, regarray_name);
10437         if (!regarray || (regarray_index >= regarray->size))
10438                 return -EINVAL;
10439
10440         r = &p->regarray_runtime[regarray->id];
10441         *value = r->regarray[regarray_index];
10442         return 0;
10443 }
10444
10445 int
10446 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10447                                    const char *regarray_name,
10448                                    uint32_t regarray_index,
10449                                    uint64_t value)
10450 {
10451         struct regarray *regarray;
10452         struct regarray_runtime *r;
10453
10454         if (!p || !regarray_name)
10455                 return -EINVAL;
10456
10457         regarray = regarray_find(p, regarray_name);
10458         if (!regarray || (regarray_index >= regarray->size))
10459                 return -EINVAL;
10460
10461         r = &p->regarray_runtime[regarray->id];
10462         r->regarray[regarray_index] = value;
10463         return 0;
10464 }
10465
10466 int
10467 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10468                               uint32_t metarray_id,
10469                               struct rte_swx_ctl_metarray_info *metarray)
10470 {
10471         struct metarray *m;
10472
10473         if (!p || !metarray)
10474                 return -EINVAL;
10475
10476         m = metarray_find_by_id(p, metarray_id);
10477         if (!m)
10478                 return -EINVAL;
10479
10480         strcpy(metarray->name, m->name);
10481         metarray->size = m->size;
10482         return 0;
10483 }
10484
10485 int
10486 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10487                               const char *name,
10488                               struct rte_meter_trtcm_params *params)
10489 {
10490         struct meter_profile *mp;
10491         int status;
10492
10493         CHECK(p, EINVAL);
10494         CHECK_NAME(name, EINVAL);
10495         CHECK(params, EINVAL);
10496         CHECK(!meter_profile_find(p, name), EEXIST);
10497
10498         /* Node allocation. */
10499         mp = calloc(1, sizeof(struct meter_profile));
10500         CHECK(mp, ENOMEM);
10501
10502         /* Node initialization. */
10503         strcpy(mp->name, name);
10504         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10505         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10506         if (status) {
10507                 free(mp);
10508                 CHECK(0, EINVAL);
10509         }
10510
10511         /* Node add to tailq. */
10512         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10513
10514         return 0;
10515 }
10516
10517 int
10518 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10519                                  const char *name)
10520 {
10521         struct meter_profile *mp;
10522
10523         CHECK(p, EINVAL);
10524         CHECK_NAME(name, EINVAL);
10525
10526         mp = meter_profile_find(p, name);
10527         CHECK(mp, EINVAL);
10528         CHECK(!mp->n_users, EBUSY);
10529
10530         /* Remove node from tailq. */
10531         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10532         free(mp);
10533
10534         return 0;
10535 }
10536
10537 int
10538 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10539                         const char *metarray_name,
10540                         uint32_t metarray_index)
10541 {
10542         struct meter_profile *mp_old;
10543         struct metarray *metarray;
10544         struct metarray_runtime *metarray_runtime;
10545         struct meter *m;
10546
10547         CHECK(p, EINVAL);
10548         CHECK_NAME(metarray_name, EINVAL);
10549
10550         metarray = metarray_find(p, metarray_name);
10551         CHECK(metarray, EINVAL);
10552         CHECK(metarray_index < metarray->size, EINVAL);
10553
10554         metarray_runtime = &p->metarray_runtime[metarray->id];
10555         m = &metarray_runtime->metarray[metarray_index];
10556         mp_old = m->profile;
10557
10558         meter_init(m);
10559
10560         mp_old->n_users--;
10561
10562         return 0;
10563 }
10564
10565 int
10566 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10567                       const char *metarray_name,
10568                       uint32_t metarray_index,
10569                       const char *profile_name)
10570 {
10571         struct meter_profile *mp, *mp_old;
10572         struct metarray *metarray;
10573         struct metarray_runtime *metarray_runtime;
10574         struct meter *m;
10575
10576         CHECK(p, EINVAL);
10577         CHECK_NAME(metarray_name, EINVAL);
10578
10579         metarray = metarray_find(p, metarray_name);
10580         CHECK(metarray, EINVAL);
10581         CHECK(metarray_index < metarray->size, EINVAL);
10582
10583         mp = meter_profile_find(p, profile_name);
10584         CHECK(mp, EINVAL);
10585
10586         metarray_runtime = &p->metarray_runtime[metarray->id];
10587         m = &metarray_runtime->metarray[metarray_index];
10588         mp_old = m->profile;
10589
10590         memset(m, 0, sizeof(struct meter));
10591         rte_meter_trtcm_config(&m->m, &mp->profile);
10592         m->profile = mp;
10593         m->color_mask = RTE_COLORS;
10594
10595         mp->n_users++;
10596         mp_old->n_users--;
10597
10598         return 0;
10599 }
10600
10601 int
10602 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10603                              const char *metarray_name,
10604                              uint32_t metarray_index,
10605                              struct rte_swx_ctl_meter_stats *stats)
10606 {
10607         struct metarray *metarray;
10608         struct metarray_runtime *metarray_runtime;
10609         struct meter *m;
10610
10611         CHECK(p, EINVAL);
10612         CHECK_NAME(metarray_name, EINVAL);
10613
10614         metarray = metarray_find(p, metarray_name);
10615         CHECK(metarray, EINVAL);
10616         CHECK(metarray_index < metarray->size, EINVAL);
10617
10618         CHECK(stats, EINVAL);
10619
10620         metarray_runtime = &p->metarray_runtime[metarray->id];
10621         m = &metarray_runtime->metarray[metarray_index];
10622
10623         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10624         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10625
10626         return 0;
10627 }