lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t selector_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107         struct selector_runtime *selector = &t->selectors[selector_id];
2108         struct selector_statistics *stats = &p->selector_stats[selector_id];
2109         uint64_t n_pkts = stats->n_pkts;
2110         int done;
2111
2112         /* Table. */
2113         done = rte_swx_table_selector_select(ts->obj,
2114                            selector->mailbox,
2115                            selector->group_id_buffer,
2116                            selector->selector_buffer,
2117                            selector->member_id_buffer);
2118         if (!done) {
2119                 /* Thread. */
2120                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2121                       p->thread_id,
2122                       selector_id);
2123
2124                 thread_yield(p);
2125                 return;
2126         }
2127
2128
2129         TRACE("[Thread %2u] selector %u\n",
2130               p->thread_id,
2131               selector_id);
2132
2133         stats->n_pkts = n_pkts + 1;
2134
2135         /* Thread. */
2136         thread_ip_inc(p);
2137 }
2138
2139 static inline void
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t learner_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146                 p->n_selectors + learner_id];
2147         struct learner_runtime *l = &t->learners[learner_id];
2148         struct learner_statistics *stats = &p->learner_stats[learner_id];
2149         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150         uint8_t *action_data;
2151         int done, hit;
2152
2153         /* Table. */
2154         time = rte_get_tsc_cycles();
2155
2156         done = rte_swx_table_learner_lookup(ts->obj,
2157                                             l->mailbox,
2158                                             time,
2159                                             l->key,
2160                                             &action_id,
2161                                             &action_data,
2162                                             &hit);
2163         if (!done) {
2164                 /* Thread. */
2165                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2166                       p->thread_id,
2167                       learner_id);
2168
2169                 thread_yield(p);
2170                 return;
2171         }
2172
2173         action_id = hit ? action_id : ts->default_action_id;
2174         action_data = hit ? action_data : ts->default_action_data;
2175         n_pkts_hit = stats->n_pkts_hit[hit];
2176         n_pkts_action = stats->n_pkts_action[action_id];
2177
2178         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2179               p->thread_id,
2180               learner_id,
2181               hit ? "hit" : "miss",
2182               (uint32_t)action_id);
2183
2184         t->action_id = action_id;
2185         t->structs[0] = action_data;
2186         t->hit = hit;
2187         t->learner_id = learner_id;
2188         t->time = time;
2189         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2191
2192         /* Thread. */
2193         thread_ip_action_call(p, t, action_id);
2194 }
2195
2196 /*
2197  * learn.
2198  */
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2201
2202 static int
2203 action_has_nbo_args(struct action *a);
2204
2205 static int
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207                       struct action *action,
2208                       char **tokens,
2209                       int n_tokens,
2210                       struct instruction *instr,
2211                       struct instruction_data *data __rte_unused)
2212 {
2213         struct action *a;
2214
2215         CHECK(action, EINVAL);
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         a = action_find(p, tokens[1]);
2219         CHECK(a, EINVAL);
2220         CHECK(!action_has_nbo_args(a), EINVAL);
2221
2222         instr->type = INSTR_LEARNER_LEARN;
2223         instr->learn.action_id = a->id;
2224
2225         return 0;
2226 }
2227
2228 static inline void
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_learn_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_inc(p);
2238 }
2239
2240 /*
2241  * forget.
2242  */
2243 static int
2244 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2245                        struct action *action,
2246                        char **tokens __rte_unused,
2247                        int n_tokens,
2248                        struct instruction *instr,
2249                        struct instruction_data *data __rte_unused)
2250 {
2251         CHECK(action, EINVAL);
2252         CHECK(n_tokens == 1, EINVAL);
2253
2254         instr->type = INSTR_LEARNER_FORGET;
2255
2256         return 0;
2257 }
2258
2259 static inline void
2260 instr_forget_exec(struct rte_swx_pipeline *p)
2261 {
2262         struct thread *t = &p->threads[p->thread_id];
2263         struct instruction *ip = t->ip;
2264
2265         __instr_forget_exec(p, t, ip);
2266
2267         /* Thread. */
2268         thread_ip_inc(p);
2269 }
2270
2271 /*
2272  * extern.
2273  */
2274 static int
2275 instr_extern_translate(struct rte_swx_pipeline *p,
2276                        struct action *action __rte_unused,
2277                        char **tokens,
2278                        int n_tokens,
2279                        struct instruction *instr,
2280                        struct instruction_data *data __rte_unused)
2281 {
2282         char *token = tokens[1];
2283
2284         CHECK(n_tokens == 2, EINVAL);
2285
2286         if (token[0] == 'e') {
2287                 struct extern_obj *obj;
2288                 struct extern_type_member_func *func;
2289
2290                 func = extern_obj_member_func_parse(p, token, &obj);
2291                 CHECK(func, EINVAL);
2292
2293                 instr->type = INSTR_EXTERN_OBJ;
2294                 instr->ext_obj.ext_obj_id = obj->id;
2295                 instr->ext_obj.func_id = func->id;
2296
2297                 return 0;
2298         }
2299
2300         if (token[0] == 'f') {
2301                 struct extern_func *func;
2302
2303                 func = extern_func_parse(p, token);
2304                 CHECK(func, EINVAL);
2305
2306                 instr->type = INSTR_EXTERN_FUNC;
2307                 instr->ext_func.ext_func_id = func->id;
2308
2309                 return 0;
2310         }
2311
2312         CHECK(0, EINVAL);
2313 }
2314
2315 static inline void
2316 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2317 {
2318         struct thread *t = &p->threads[p->thread_id];
2319         struct instruction *ip = t->ip;
2320         uint32_t done;
2321
2322         /* Extern object member function execute. */
2323         done = __instr_extern_obj_exec(p, t, ip);
2324
2325         /* Thread. */
2326         thread_ip_inc_cond(t, done);
2327         thread_yield_cond(p, done ^ 1);
2328 }
2329
2330 static inline void
2331 instr_extern_func_exec(struct rte_swx_pipeline *p)
2332 {
2333         struct thread *t = &p->threads[p->thread_id];
2334         struct instruction *ip = t->ip;
2335         uint32_t done;
2336
2337         /* Extern function execute. */
2338         done = __instr_extern_func_exec(p, t, ip);
2339
2340         /* Thread. */
2341         thread_ip_inc_cond(t, done);
2342         thread_yield_cond(p, done ^ 1);
2343 }
2344
2345 /*
2346  * mov.
2347  */
2348 static int
2349 instr_mov_translate(struct rte_swx_pipeline *p,
2350                     struct action *action,
2351                     char **tokens,
2352                     int n_tokens,
2353                     struct instruction *instr,
2354                     struct instruction_data *data __rte_unused)
2355 {
2356         char *dst = tokens[1], *src = tokens[2];
2357         struct field *fdst, *fsrc;
2358         uint64_t src_val;
2359         uint32_t dst_struct_id = 0, src_struct_id = 0;
2360
2361         CHECK(n_tokens == 3, EINVAL);
2362
2363         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2364         CHECK(fdst, EINVAL);
2365         CHECK(!fdst->var_size, EINVAL);
2366
2367         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2368         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2369         if (fsrc) {
2370                 CHECK(!fsrc->var_size, EINVAL);
2371
2372                 instr->type = INSTR_MOV;
2373                 if (dst[0] != 'h' && src[0] == 'h')
2374                         instr->type = INSTR_MOV_MH;
2375                 if (dst[0] == 'h' && src[0] != 'h')
2376                         instr->type = INSTR_MOV_HM;
2377                 if (dst[0] == 'h' && src[0] == 'h')
2378                         instr->type = INSTR_MOV_HH;
2379
2380                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2381                 instr->mov.dst.n_bits = fdst->n_bits;
2382                 instr->mov.dst.offset = fdst->offset / 8;
2383                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2384                 instr->mov.src.n_bits = fsrc->n_bits;
2385                 instr->mov.src.offset = fsrc->offset / 8;
2386                 return 0;
2387         }
2388
2389         /* MOV_I. */
2390         src_val = strtoull(src, &src, 0);
2391         CHECK(!src[0], EINVAL);
2392
2393         if (dst[0] == 'h')
2394                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2395
2396         instr->type = INSTR_MOV_I;
2397         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2398         instr->mov.dst.n_bits = fdst->n_bits;
2399         instr->mov.dst.offset = fdst->offset / 8;
2400         instr->mov.src_val = src_val;
2401         return 0;
2402 }
2403
2404 static inline void
2405 instr_mov_exec(struct rte_swx_pipeline *p)
2406 {
2407         struct thread *t = &p->threads[p->thread_id];
2408         struct instruction *ip = t->ip;
2409
2410         __instr_mov_exec(p, t, ip);
2411
2412         /* Thread. */
2413         thread_ip_inc(p);
2414 }
2415
2416 static inline void
2417 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2418 {
2419         struct thread *t = &p->threads[p->thread_id];
2420         struct instruction *ip = t->ip;
2421
2422         __instr_mov_mh_exec(p, t, ip);
2423
2424         /* Thread. */
2425         thread_ip_inc(p);
2426 }
2427
2428 static inline void
2429 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2430 {
2431         struct thread *t = &p->threads[p->thread_id];
2432         struct instruction *ip = t->ip;
2433
2434         __instr_mov_hm_exec(p, t, ip);
2435
2436         /* Thread. */
2437         thread_ip_inc(p);
2438 }
2439
2440 static inline void
2441 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2442 {
2443         struct thread *t = &p->threads[p->thread_id];
2444         struct instruction *ip = t->ip;
2445
2446         __instr_mov_hh_exec(p, t, ip);
2447
2448         /* Thread. */
2449         thread_ip_inc(p);
2450 }
2451
2452 static inline void
2453 instr_mov_i_exec(struct rte_swx_pipeline *p)
2454 {
2455         struct thread *t = &p->threads[p->thread_id];
2456         struct instruction *ip = t->ip;
2457
2458         __instr_mov_i_exec(p, t, ip);
2459
2460         /* Thread. */
2461         thread_ip_inc(p);
2462 }
2463
2464 /*
2465  * dma.
2466  */
2467 static inline void
2468 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2469 {
2470         struct thread *t = &p->threads[p->thread_id];
2471         struct instruction *ip = t->ip;
2472
2473         __instr_dma_ht_exec(p, t, ip);
2474
2475         /* Thread. */
2476         thread_ip_inc(p);
2477 }
2478
2479 static inline void
2480 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2481 {
2482         struct thread *t = &p->threads[p->thread_id];
2483         struct instruction *ip = t->ip;
2484
2485         __instr_dma_ht2_exec(p, t, ip);
2486
2487         /* Thread. */
2488         thread_ip_inc(p);
2489 }
2490
2491 static inline void
2492 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2493 {
2494         struct thread *t = &p->threads[p->thread_id];
2495         struct instruction *ip = t->ip;
2496
2497         __instr_dma_ht3_exec(p, t, ip);
2498
2499         /* Thread. */
2500         thread_ip_inc(p);
2501 }
2502
2503 static inline void
2504 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2505 {
2506         struct thread *t = &p->threads[p->thread_id];
2507         struct instruction *ip = t->ip;
2508
2509         __instr_dma_ht4_exec(p, t, ip);
2510
2511         /* Thread. */
2512         thread_ip_inc(p);
2513 }
2514
2515 static inline void
2516 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2517 {
2518         struct thread *t = &p->threads[p->thread_id];
2519         struct instruction *ip = t->ip;
2520
2521         __instr_dma_ht5_exec(p, t, ip);
2522
2523         /* Thread. */
2524         thread_ip_inc(p);
2525 }
2526
2527 static inline void
2528 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2529 {
2530         struct thread *t = &p->threads[p->thread_id];
2531         struct instruction *ip = t->ip;
2532
2533         __instr_dma_ht6_exec(p, t, ip);
2534
2535         /* Thread. */
2536         thread_ip_inc(p);
2537 }
2538
2539 static inline void
2540 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2541 {
2542         struct thread *t = &p->threads[p->thread_id];
2543         struct instruction *ip = t->ip;
2544
2545         __instr_dma_ht7_exec(p, t, ip);
2546
2547         /* Thread. */
2548         thread_ip_inc(p);
2549 }
2550
2551 static inline void
2552 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2553 {
2554         struct thread *t = &p->threads[p->thread_id];
2555         struct instruction *ip = t->ip;
2556
2557         __instr_dma_ht8_exec(p, t, ip);
2558
2559         /* Thread. */
2560         thread_ip_inc(p);
2561 }
2562
2563 /*
2564  * alu.
2565  */
2566 static int
2567 instr_alu_add_translate(struct rte_swx_pipeline *p,
2568                         struct action *action,
2569                         char **tokens,
2570                         int n_tokens,
2571                         struct instruction *instr,
2572                         struct instruction_data *data __rte_unused)
2573 {
2574         char *dst = tokens[1], *src = tokens[2];
2575         struct field *fdst, *fsrc;
2576         uint64_t src_val;
2577         uint32_t dst_struct_id = 0, src_struct_id = 0;
2578
2579         CHECK(n_tokens == 3, EINVAL);
2580
2581         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2582         CHECK(fdst, EINVAL);
2583         CHECK(!fdst->var_size, EINVAL);
2584
2585         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2586         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2587         if (fsrc) {
2588                 CHECK(!fsrc->var_size, EINVAL);
2589
2590                 instr->type = INSTR_ALU_ADD;
2591                 if (dst[0] == 'h' && src[0] != 'h')
2592                         instr->type = INSTR_ALU_ADD_HM;
2593                 if (dst[0] != 'h' && src[0] == 'h')
2594                         instr->type = INSTR_ALU_ADD_MH;
2595                 if (dst[0] == 'h' && src[0] == 'h')
2596                         instr->type = INSTR_ALU_ADD_HH;
2597
2598                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2599                 instr->alu.dst.n_bits = fdst->n_bits;
2600                 instr->alu.dst.offset = fdst->offset / 8;
2601                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2602                 instr->alu.src.n_bits = fsrc->n_bits;
2603                 instr->alu.src.offset = fsrc->offset / 8;
2604                 return 0;
2605         }
2606
2607         /* ADD_MI, ADD_HI. */
2608         src_val = strtoull(src, &src, 0);
2609         CHECK(!src[0], EINVAL);
2610
2611         instr->type = INSTR_ALU_ADD_MI;
2612         if (dst[0] == 'h')
2613                 instr->type = INSTR_ALU_ADD_HI;
2614
2615         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2616         instr->alu.dst.n_bits = fdst->n_bits;
2617         instr->alu.dst.offset = fdst->offset / 8;
2618         instr->alu.src_val = src_val;
2619         return 0;
2620 }
2621
2622 static int
2623 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2624                         struct action *action,
2625                         char **tokens,
2626                         int n_tokens,
2627                         struct instruction *instr,
2628                         struct instruction_data *data __rte_unused)
2629 {
2630         char *dst = tokens[1], *src = tokens[2];
2631         struct field *fdst, *fsrc;
2632         uint64_t src_val;
2633         uint32_t dst_struct_id = 0, src_struct_id = 0;
2634
2635         CHECK(n_tokens == 3, EINVAL);
2636
2637         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2638         CHECK(fdst, EINVAL);
2639         CHECK(!fdst->var_size, EINVAL);
2640
2641         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2642         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2643         if (fsrc) {
2644                 CHECK(!fsrc->var_size, EINVAL);
2645
2646                 instr->type = INSTR_ALU_SUB;
2647                 if (dst[0] == 'h' && src[0] != 'h')
2648                         instr->type = INSTR_ALU_SUB_HM;
2649                 if (dst[0] != 'h' && src[0] == 'h')
2650                         instr->type = INSTR_ALU_SUB_MH;
2651                 if (dst[0] == 'h' && src[0] == 'h')
2652                         instr->type = INSTR_ALU_SUB_HH;
2653
2654                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2655                 instr->alu.dst.n_bits = fdst->n_bits;
2656                 instr->alu.dst.offset = fdst->offset / 8;
2657                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2658                 instr->alu.src.n_bits = fsrc->n_bits;
2659                 instr->alu.src.offset = fsrc->offset / 8;
2660                 return 0;
2661         }
2662
2663         /* SUB_MI, SUB_HI. */
2664         src_val = strtoull(src, &src, 0);
2665         CHECK(!src[0], EINVAL);
2666
2667         instr->type = INSTR_ALU_SUB_MI;
2668         if (dst[0] == 'h')
2669                 instr->type = INSTR_ALU_SUB_HI;
2670
2671         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2672         instr->alu.dst.n_bits = fdst->n_bits;
2673         instr->alu.dst.offset = fdst->offset / 8;
2674         instr->alu.src_val = src_val;
2675         return 0;
2676 }
2677
2678 static int
2679 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2680                           struct action *action __rte_unused,
2681                           char **tokens,
2682                           int n_tokens,
2683                           struct instruction *instr,
2684                           struct instruction_data *data __rte_unused)
2685 {
2686         char *dst = tokens[1], *src = tokens[2];
2687         struct header *hdst, *hsrc;
2688         struct field *fdst, *fsrc;
2689
2690         CHECK(n_tokens == 3, EINVAL);
2691
2692         fdst = header_field_parse(p, dst, &hdst);
2693         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2694         CHECK(!fdst->var_size, EINVAL);
2695
2696         /* CKADD_FIELD. */
2697         fsrc = header_field_parse(p, src, &hsrc);
2698         if (fsrc) {
2699                 CHECK(!fsrc->var_size, EINVAL);
2700
2701                 instr->type = INSTR_ALU_CKADD_FIELD;
2702                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2703                 instr->alu.dst.n_bits = fdst->n_bits;
2704                 instr->alu.dst.offset = fdst->offset / 8;
2705                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2706                 instr->alu.src.n_bits = fsrc->n_bits;
2707                 instr->alu.src.offset = fsrc->offset / 8;
2708                 return 0;
2709         }
2710
2711         /* CKADD_STRUCT, CKADD_STRUCT20. */
2712         hsrc = header_parse(p, src);
2713         CHECK(hsrc, EINVAL);
2714         CHECK(!hsrc->st->var_size, EINVAL);
2715
2716         instr->type = INSTR_ALU_CKADD_STRUCT;
2717         if ((hsrc->st->n_bits / 8) == 20)
2718                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2719
2720         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2721         instr->alu.dst.n_bits = fdst->n_bits;
2722         instr->alu.dst.offset = fdst->offset / 8;
2723         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2724         instr->alu.src.n_bits = hsrc->st->n_bits;
2725         instr->alu.src.offset = 0; /* Unused. */
2726         return 0;
2727 }
2728
2729 static int
2730 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2731                           struct action *action __rte_unused,
2732                           char **tokens,
2733                           int n_tokens,
2734                           struct instruction *instr,
2735                           struct instruction_data *data __rte_unused)
2736 {
2737         char *dst = tokens[1], *src = tokens[2];
2738         struct header *hdst, *hsrc;
2739         struct field *fdst, *fsrc;
2740
2741         CHECK(n_tokens == 3, EINVAL);
2742
2743         fdst = header_field_parse(p, dst, &hdst);
2744         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2745         CHECK(!fdst->var_size, EINVAL);
2746
2747         fsrc = header_field_parse(p, src, &hsrc);
2748         CHECK(fsrc, EINVAL);
2749         CHECK(!fsrc->var_size, EINVAL);
2750
2751         instr->type = INSTR_ALU_CKSUB_FIELD;
2752         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2753         instr->alu.dst.n_bits = fdst->n_bits;
2754         instr->alu.dst.offset = fdst->offset / 8;
2755         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2756         instr->alu.src.n_bits = fsrc->n_bits;
2757         instr->alu.src.offset = fsrc->offset / 8;
2758         return 0;
2759 }
2760
2761 static int
2762 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2763                         struct action *action,
2764                         char **tokens,
2765                         int n_tokens,
2766                         struct instruction *instr,
2767                         struct instruction_data *data __rte_unused)
2768 {
2769         char *dst = tokens[1], *src = tokens[2];
2770         struct field *fdst, *fsrc;
2771         uint64_t src_val;
2772         uint32_t dst_struct_id = 0, src_struct_id = 0;
2773
2774         CHECK(n_tokens == 3, EINVAL);
2775
2776         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2777         CHECK(fdst, EINVAL);
2778         CHECK(!fdst->var_size, EINVAL);
2779
2780         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2781         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2782         if (fsrc) {
2783                 CHECK(!fsrc->var_size, EINVAL);
2784
2785                 instr->type = INSTR_ALU_SHL;
2786                 if (dst[0] == 'h' && src[0] != 'h')
2787                         instr->type = INSTR_ALU_SHL_HM;
2788                 if (dst[0] != 'h' && src[0] == 'h')
2789                         instr->type = INSTR_ALU_SHL_MH;
2790                 if (dst[0] == 'h' && src[0] == 'h')
2791                         instr->type = INSTR_ALU_SHL_HH;
2792
2793                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2794                 instr->alu.dst.n_bits = fdst->n_bits;
2795                 instr->alu.dst.offset = fdst->offset / 8;
2796                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2797                 instr->alu.src.n_bits = fsrc->n_bits;
2798                 instr->alu.src.offset = fsrc->offset / 8;
2799                 return 0;
2800         }
2801
2802         /* SHL_MI, SHL_HI. */
2803         src_val = strtoull(src, &src, 0);
2804         CHECK(!src[0], EINVAL);
2805
2806         instr->type = INSTR_ALU_SHL_MI;
2807         if (dst[0] == 'h')
2808                 instr->type = INSTR_ALU_SHL_HI;
2809
2810         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2811         instr->alu.dst.n_bits = fdst->n_bits;
2812         instr->alu.dst.offset = fdst->offset / 8;
2813         instr->alu.src_val = src_val;
2814         return 0;
2815 }
2816
2817 static int
2818 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2819                         struct action *action,
2820                         char **tokens,
2821                         int n_tokens,
2822                         struct instruction *instr,
2823                         struct instruction_data *data __rte_unused)
2824 {
2825         char *dst = tokens[1], *src = tokens[2];
2826         struct field *fdst, *fsrc;
2827         uint64_t src_val;
2828         uint32_t dst_struct_id = 0, src_struct_id = 0;
2829
2830         CHECK(n_tokens == 3, EINVAL);
2831
2832         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2833         CHECK(fdst, EINVAL);
2834         CHECK(!fdst->var_size, EINVAL);
2835
2836         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2837         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2838         if (fsrc) {
2839                 CHECK(!fsrc->var_size, EINVAL);
2840
2841                 instr->type = INSTR_ALU_SHR;
2842                 if (dst[0] == 'h' && src[0] != 'h')
2843                         instr->type = INSTR_ALU_SHR_HM;
2844                 if (dst[0] != 'h' && src[0] == 'h')
2845                         instr->type = INSTR_ALU_SHR_MH;
2846                 if (dst[0] == 'h' && src[0] == 'h')
2847                         instr->type = INSTR_ALU_SHR_HH;
2848
2849                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2850                 instr->alu.dst.n_bits = fdst->n_bits;
2851                 instr->alu.dst.offset = fdst->offset / 8;
2852                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2853                 instr->alu.src.n_bits = fsrc->n_bits;
2854                 instr->alu.src.offset = fsrc->offset / 8;
2855                 return 0;
2856         }
2857
2858         /* SHR_MI, SHR_HI. */
2859         src_val = strtoull(src, &src, 0);
2860         CHECK(!src[0], EINVAL);
2861
2862         instr->type = INSTR_ALU_SHR_MI;
2863         if (dst[0] == 'h')
2864                 instr->type = INSTR_ALU_SHR_HI;
2865
2866         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2867         instr->alu.dst.n_bits = fdst->n_bits;
2868         instr->alu.dst.offset = fdst->offset / 8;
2869         instr->alu.src_val = src_val;
2870         return 0;
2871 }
2872
2873 static int
2874 instr_alu_and_translate(struct rte_swx_pipeline *p,
2875                         struct action *action,
2876                         char **tokens,
2877                         int n_tokens,
2878                         struct instruction *instr,
2879                         struct instruction_data *data __rte_unused)
2880 {
2881         char *dst = tokens[1], *src = tokens[2];
2882         struct field *fdst, *fsrc;
2883         uint64_t src_val;
2884         uint32_t dst_struct_id = 0, src_struct_id = 0;
2885
2886         CHECK(n_tokens == 3, EINVAL);
2887
2888         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2889         CHECK(fdst, EINVAL);
2890         CHECK(!fdst->var_size, EINVAL);
2891
2892         /* AND, AND_MH, AND_HM, AND_HH. */
2893         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2894         if (fsrc) {
2895                 CHECK(!fsrc->var_size, EINVAL);
2896
2897                 instr->type = INSTR_ALU_AND;
2898                 if (dst[0] != 'h' && src[0] == 'h')
2899                         instr->type = INSTR_ALU_AND_MH;
2900                 if (dst[0] == 'h' && src[0] != 'h')
2901                         instr->type = INSTR_ALU_AND_HM;
2902                 if (dst[0] == 'h' && src[0] == 'h')
2903                         instr->type = INSTR_ALU_AND_HH;
2904
2905                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2906                 instr->alu.dst.n_bits = fdst->n_bits;
2907                 instr->alu.dst.offset = fdst->offset / 8;
2908                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2909                 instr->alu.src.n_bits = fsrc->n_bits;
2910                 instr->alu.src.offset = fsrc->offset / 8;
2911                 return 0;
2912         }
2913
2914         /* AND_I. */
2915         src_val = strtoull(src, &src, 0);
2916         CHECK(!src[0], EINVAL);
2917
2918         if (dst[0] == 'h')
2919                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2920
2921         instr->type = INSTR_ALU_AND_I;
2922         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2923         instr->alu.dst.n_bits = fdst->n_bits;
2924         instr->alu.dst.offset = fdst->offset / 8;
2925         instr->alu.src_val = src_val;
2926         return 0;
2927 }
2928
2929 static int
2930 instr_alu_or_translate(struct rte_swx_pipeline *p,
2931                        struct action *action,
2932                        char **tokens,
2933                        int n_tokens,
2934                        struct instruction *instr,
2935                        struct instruction_data *data __rte_unused)
2936 {
2937         char *dst = tokens[1], *src = tokens[2];
2938         struct field *fdst, *fsrc;
2939         uint64_t src_val;
2940         uint32_t dst_struct_id = 0, src_struct_id = 0;
2941
2942         CHECK(n_tokens == 3, EINVAL);
2943
2944         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2945         CHECK(fdst, EINVAL);
2946         CHECK(!fdst->var_size, EINVAL);
2947
2948         /* OR, OR_MH, OR_HM, OR_HH. */
2949         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2950         if (fsrc) {
2951                 CHECK(!fsrc->var_size, EINVAL);
2952
2953                 instr->type = INSTR_ALU_OR;
2954                 if (dst[0] != 'h' && src[0] == 'h')
2955                         instr->type = INSTR_ALU_OR_MH;
2956                 if (dst[0] == 'h' && src[0] != 'h')
2957                         instr->type = INSTR_ALU_OR_HM;
2958                 if (dst[0] == 'h' && src[0] == 'h')
2959                         instr->type = INSTR_ALU_OR_HH;
2960
2961                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2962                 instr->alu.dst.n_bits = fdst->n_bits;
2963                 instr->alu.dst.offset = fdst->offset / 8;
2964                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2965                 instr->alu.src.n_bits = fsrc->n_bits;
2966                 instr->alu.src.offset = fsrc->offset / 8;
2967                 return 0;
2968         }
2969
2970         /* OR_I. */
2971         src_val = strtoull(src, &src, 0);
2972         CHECK(!src[0], EINVAL);
2973
2974         if (dst[0] == 'h')
2975                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2976
2977         instr->type = INSTR_ALU_OR_I;
2978         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2979         instr->alu.dst.n_bits = fdst->n_bits;
2980         instr->alu.dst.offset = fdst->offset / 8;
2981         instr->alu.src_val = src_val;
2982         return 0;
2983 }
2984
2985 static int
2986 instr_alu_xor_translate(struct rte_swx_pipeline *p,
2987                         struct action *action,
2988                         char **tokens,
2989                         int n_tokens,
2990                         struct instruction *instr,
2991                         struct instruction_data *data __rte_unused)
2992 {
2993         char *dst = tokens[1], *src = tokens[2];
2994         struct field *fdst, *fsrc;
2995         uint64_t src_val;
2996         uint32_t dst_struct_id = 0, src_struct_id = 0;
2997
2998         CHECK(n_tokens == 3, EINVAL);
2999
3000         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3001         CHECK(fdst, EINVAL);
3002         CHECK(!fdst->var_size, EINVAL);
3003
3004         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3006         if (fsrc) {
3007                 CHECK(!fsrc->var_size, EINVAL);
3008
3009                 instr->type = INSTR_ALU_XOR;
3010                 if (dst[0] != 'h' && src[0] == 'h')
3011                         instr->type = INSTR_ALU_XOR_MH;
3012                 if (dst[0] == 'h' && src[0] != 'h')
3013                         instr->type = INSTR_ALU_XOR_HM;
3014                 if (dst[0] == 'h' && src[0] == 'h')
3015                         instr->type = INSTR_ALU_XOR_HH;
3016
3017                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3018                 instr->alu.dst.n_bits = fdst->n_bits;
3019                 instr->alu.dst.offset = fdst->offset / 8;
3020                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3021                 instr->alu.src.n_bits = fsrc->n_bits;
3022                 instr->alu.src.offset = fsrc->offset / 8;
3023                 return 0;
3024         }
3025
3026         /* XOR_I. */
3027         src_val = strtoull(src, &src, 0);
3028         CHECK(!src[0], EINVAL);
3029
3030         if (dst[0] == 'h')
3031                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3032
3033         instr->type = INSTR_ALU_XOR_I;
3034         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3035         instr->alu.dst.n_bits = fdst->n_bits;
3036         instr->alu.dst.offset = fdst->offset / 8;
3037         instr->alu.src_val = src_val;
3038         return 0;
3039 }
3040
3041 static inline void
3042 instr_alu_add_exec(struct rte_swx_pipeline *p)
3043 {
3044         struct thread *t = &p->threads[p->thread_id];
3045         struct instruction *ip = t->ip;
3046
3047         TRACE("[Thread %2u] add\n", p->thread_id);
3048
3049         /* Structs. */
3050         ALU(t, ip, +);
3051
3052         /* Thread. */
3053         thread_ip_inc(p);
3054 }
3055
3056 static inline void
3057 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3058 {
3059         struct thread *t = &p->threads[p->thread_id];
3060         struct instruction *ip = t->ip;
3061
3062         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3063
3064         /* Structs. */
3065         ALU_MH(t, ip, +);
3066
3067         /* Thread. */
3068         thread_ip_inc(p);
3069 }
3070
3071 static inline void
3072 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3073 {
3074         struct thread *t = &p->threads[p->thread_id];
3075         struct instruction *ip = t->ip;
3076
3077         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3078
3079         /* Structs. */
3080         ALU_HM(t, ip, +);
3081
3082         /* Thread. */
3083         thread_ip_inc(p);
3084 }
3085
3086 static inline void
3087 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3088 {
3089         struct thread *t = &p->threads[p->thread_id];
3090         struct instruction *ip = t->ip;
3091
3092         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3093
3094         /* Structs. */
3095         ALU_HH(t, ip, +);
3096
3097         /* Thread. */
3098         thread_ip_inc(p);
3099 }
3100
3101 static inline void
3102 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3103 {
3104         struct thread *t = &p->threads[p->thread_id];
3105         struct instruction *ip = t->ip;
3106
3107         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3108
3109         /* Structs. */
3110         ALU_MI(t, ip, +);
3111
3112         /* Thread. */
3113         thread_ip_inc(p);
3114 }
3115
3116 static inline void
3117 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3118 {
3119         struct thread *t = &p->threads[p->thread_id];
3120         struct instruction *ip = t->ip;
3121
3122         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3123
3124         /* Structs. */
3125         ALU_HI(t, ip, +);
3126
3127         /* Thread. */
3128         thread_ip_inc(p);
3129 }
3130
3131 static inline void
3132 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3133 {
3134         struct thread *t = &p->threads[p->thread_id];
3135         struct instruction *ip = t->ip;
3136
3137         TRACE("[Thread %2u] sub\n", p->thread_id);
3138
3139         /* Structs. */
3140         ALU(t, ip, -);
3141
3142         /* Thread. */
3143         thread_ip_inc(p);
3144 }
3145
3146 static inline void
3147 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3148 {
3149         struct thread *t = &p->threads[p->thread_id];
3150         struct instruction *ip = t->ip;
3151
3152         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3153
3154         /* Structs. */
3155         ALU_MH(t, ip, -);
3156
3157         /* Thread. */
3158         thread_ip_inc(p);
3159 }
3160
3161 static inline void
3162 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3163 {
3164         struct thread *t = &p->threads[p->thread_id];
3165         struct instruction *ip = t->ip;
3166
3167         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3168
3169         /* Structs. */
3170         ALU_HM(t, ip, -);
3171
3172         /* Thread. */
3173         thread_ip_inc(p);
3174 }
3175
3176 static inline void
3177 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3178 {
3179         struct thread *t = &p->threads[p->thread_id];
3180         struct instruction *ip = t->ip;
3181
3182         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3183
3184         /* Structs. */
3185         ALU_HH(t, ip, -);
3186
3187         /* Thread. */
3188         thread_ip_inc(p);
3189 }
3190
3191 static inline void
3192 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3193 {
3194         struct thread *t = &p->threads[p->thread_id];
3195         struct instruction *ip = t->ip;
3196
3197         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3198
3199         /* Structs. */
3200         ALU_MI(t, ip, -);
3201
3202         /* Thread. */
3203         thread_ip_inc(p);
3204 }
3205
3206 static inline void
3207 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3208 {
3209         struct thread *t = &p->threads[p->thread_id];
3210         struct instruction *ip = t->ip;
3211
3212         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3213
3214         /* Structs. */
3215         ALU_HI(t, ip, -);
3216
3217         /* Thread. */
3218         thread_ip_inc(p);
3219 }
3220
3221 static inline void
3222 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3223 {
3224         struct thread *t = &p->threads[p->thread_id];
3225         struct instruction *ip = t->ip;
3226
3227         TRACE("[Thread %2u] shl\n", p->thread_id);
3228
3229         /* Structs. */
3230         ALU(t, ip, <<);
3231
3232         /* Thread. */
3233         thread_ip_inc(p);
3234 }
3235
3236 static inline void
3237 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3238 {
3239         struct thread *t = &p->threads[p->thread_id];
3240         struct instruction *ip = t->ip;
3241
3242         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3243
3244         /* Structs. */
3245         ALU_MH(t, ip, <<);
3246
3247         /* Thread. */
3248         thread_ip_inc(p);
3249 }
3250
3251 static inline void
3252 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3253 {
3254         struct thread *t = &p->threads[p->thread_id];
3255         struct instruction *ip = t->ip;
3256
3257         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3258
3259         /* Structs. */
3260         ALU_HM(t, ip, <<);
3261
3262         /* Thread. */
3263         thread_ip_inc(p);
3264 }
3265
3266 static inline void
3267 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3268 {
3269         struct thread *t = &p->threads[p->thread_id];
3270         struct instruction *ip = t->ip;
3271
3272         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3273
3274         /* Structs. */
3275         ALU_HH(t, ip, <<);
3276
3277         /* Thread. */
3278         thread_ip_inc(p);
3279 }
3280
3281 static inline void
3282 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3283 {
3284         struct thread *t = &p->threads[p->thread_id];
3285         struct instruction *ip = t->ip;
3286
3287         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3288
3289         /* Structs. */
3290         ALU_MI(t, ip, <<);
3291
3292         /* Thread. */
3293         thread_ip_inc(p);
3294 }
3295
3296 static inline void
3297 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3298 {
3299         struct thread *t = &p->threads[p->thread_id];
3300         struct instruction *ip = t->ip;
3301
3302         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3303
3304         /* Structs. */
3305         ALU_HI(t, ip, <<);
3306
3307         /* Thread. */
3308         thread_ip_inc(p);
3309 }
3310
3311 static inline void
3312 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3313 {
3314         struct thread *t = &p->threads[p->thread_id];
3315         struct instruction *ip = t->ip;
3316
3317         TRACE("[Thread %2u] shr\n", p->thread_id);
3318
3319         /* Structs. */
3320         ALU(t, ip, >>);
3321
3322         /* Thread. */
3323         thread_ip_inc(p);
3324 }
3325
3326 static inline void
3327 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3328 {
3329         struct thread *t = &p->threads[p->thread_id];
3330         struct instruction *ip = t->ip;
3331
3332         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3333
3334         /* Structs. */
3335         ALU_MH(t, ip, >>);
3336
3337         /* Thread. */
3338         thread_ip_inc(p);
3339 }
3340
3341 static inline void
3342 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3343 {
3344         struct thread *t = &p->threads[p->thread_id];
3345         struct instruction *ip = t->ip;
3346
3347         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3348
3349         /* Structs. */
3350         ALU_HM(t, ip, >>);
3351
3352         /* Thread. */
3353         thread_ip_inc(p);
3354 }
3355
3356 static inline void
3357 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3358 {
3359         struct thread *t = &p->threads[p->thread_id];
3360         struct instruction *ip = t->ip;
3361
3362         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3363
3364         /* Structs. */
3365         ALU_HH(t, ip, >>);
3366
3367         /* Thread. */
3368         thread_ip_inc(p);
3369 }
3370
3371 static inline void
3372 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3373 {
3374         struct thread *t = &p->threads[p->thread_id];
3375         struct instruction *ip = t->ip;
3376
3377         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3378
3379         /* Structs. */
3380         ALU_MI(t, ip, >>);
3381
3382         /* Thread. */
3383         thread_ip_inc(p);
3384 }
3385
3386 static inline void
3387 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3388 {
3389         struct thread *t = &p->threads[p->thread_id];
3390         struct instruction *ip = t->ip;
3391
3392         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3393
3394         /* Structs. */
3395         ALU_HI(t, ip, >>);
3396
3397         /* Thread. */
3398         thread_ip_inc(p);
3399 }
3400
3401 static inline void
3402 instr_alu_and_exec(struct rte_swx_pipeline *p)
3403 {
3404         struct thread *t = &p->threads[p->thread_id];
3405         struct instruction *ip = t->ip;
3406
3407         TRACE("[Thread %2u] and\n", p->thread_id);
3408
3409         /* Structs. */
3410         ALU(t, ip, &);
3411
3412         /* Thread. */
3413         thread_ip_inc(p);
3414 }
3415
3416 static inline void
3417 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3418 {
3419         struct thread *t = &p->threads[p->thread_id];
3420         struct instruction *ip = t->ip;
3421
3422         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3423
3424         /* Structs. */
3425         ALU_MH(t, ip, &);
3426
3427         /* Thread. */
3428         thread_ip_inc(p);
3429 }
3430
3431 static inline void
3432 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3433 {
3434         struct thread *t = &p->threads[p->thread_id];
3435         struct instruction *ip = t->ip;
3436
3437         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3438
3439         /* Structs. */
3440         ALU_HM_FAST(t, ip, &);
3441
3442         /* Thread. */
3443         thread_ip_inc(p);
3444 }
3445
3446 static inline void
3447 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3448 {
3449         struct thread *t = &p->threads[p->thread_id];
3450         struct instruction *ip = t->ip;
3451
3452         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3453
3454         /* Structs. */
3455         ALU_HH_FAST(t, ip, &);
3456
3457         /* Thread. */
3458         thread_ip_inc(p);
3459 }
3460
3461 static inline void
3462 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3463 {
3464         struct thread *t = &p->threads[p->thread_id];
3465         struct instruction *ip = t->ip;
3466
3467         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3468
3469         /* Structs. */
3470         ALU_I(t, ip, &);
3471
3472         /* Thread. */
3473         thread_ip_inc(p);
3474 }
3475
3476 static inline void
3477 instr_alu_or_exec(struct rte_swx_pipeline *p)
3478 {
3479         struct thread *t = &p->threads[p->thread_id];
3480         struct instruction *ip = t->ip;
3481
3482         TRACE("[Thread %2u] or\n", p->thread_id);
3483
3484         /* Structs. */
3485         ALU(t, ip, |);
3486
3487         /* Thread. */
3488         thread_ip_inc(p);
3489 }
3490
3491 static inline void
3492 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3493 {
3494         struct thread *t = &p->threads[p->thread_id];
3495         struct instruction *ip = t->ip;
3496
3497         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3498
3499         /* Structs. */
3500         ALU_MH(t, ip, |);
3501
3502         /* Thread. */
3503         thread_ip_inc(p);
3504 }
3505
3506 static inline void
3507 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3508 {
3509         struct thread *t = &p->threads[p->thread_id];
3510         struct instruction *ip = t->ip;
3511
3512         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3513
3514         /* Structs. */
3515         ALU_HM_FAST(t, ip, |);
3516
3517         /* Thread. */
3518         thread_ip_inc(p);
3519 }
3520
3521 static inline void
3522 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3523 {
3524         struct thread *t = &p->threads[p->thread_id];
3525         struct instruction *ip = t->ip;
3526
3527         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3528
3529         /* Structs. */
3530         ALU_HH_FAST(t, ip, |);
3531
3532         /* Thread. */
3533         thread_ip_inc(p);
3534 }
3535
3536 static inline void
3537 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3538 {
3539         struct thread *t = &p->threads[p->thread_id];
3540         struct instruction *ip = t->ip;
3541
3542         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3543
3544         /* Structs. */
3545         ALU_I(t, ip, |);
3546
3547         /* Thread. */
3548         thread_ip_inc(p);
3549 }
3550
3551 static inline void
3552 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3553 {
3554         struct thread *t = &p->threads[p->thread_id];
3555         struct instruction *ip = t->ip;
3556
3557         TRACE("[Thread %2u] xor\n", p->thread_id);
3558
3559         /* Structs. */
3560         ALU(t, ip, ^);
3561
3562         /* Thread. */
3563         thread_ip_inc(p);
3564 }
3565
3566 static inline void
3567 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3568 {
3569         struct thread *t = &p->threads[p->thread_id];
3570         struct instruction *ip = t->ip;
3571
3572         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3573
3574         /* Structs. */
3575         ALU_MH(t, ip, ^);
3576
3577         /* Thread. */
3578         thread_ip_inc(p);
3579 }
3580
3581 static inline void
3582 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3583 {
3584         struct thread *t = &p->threads[p->thread_id];
3585         struct instruction *ip = t->ip;
3586
3587         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3588
3589         /* Structs. */
3590         ALU_HM_FAST(t, ip, ^);
3591
3592         /* Thread. */
3593         thread_ip_inc(p);
3594 }
3595
3596 static inline void
3597 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3598 {
3599         struct thread *t = &p->threads[p->thread_id];
3600         struct instruction *ip = t->ip;
3601
3602         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3603
3604         /* Structs. */
3605         ALU_HH_FAST(t, ip, ^);
3606
3607         /* Thread. */
3608         thread_ip_inc(p);
3609 }
3610
3611 static inline void
3612 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3613 {
3614         struct thread *t = &p->threads[p->thread_id];
3615         struct instruction *ip = t->ip;
3616
3617         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3618
3619         /* Structs. */
3620         ALU_I(t, ip, ^);
3621
3622         /* Thread. */
3623         thread_ip_inc(p);
3624 }
3625
3626 static inline void
3627 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3628 {
3629         struct thread *t = &p->threads[p->thread_id];
3630         struct instruction *ip = t->ip;
3631         uint8_t *dst_struct, *src_struct;
3632         uint16_t *dst16_ptr, dst;
3633         uint64_t *src64_ptr, src64, src64_mask, src;
3634         uint64_t r;
3635
3636         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3637
3638         /* Structs. */
3639         dst_struct = t->structs[ip->alu.dst.struct_id];
3640         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3641         dst = *dst16_ptr;
3642
3643         src_struct = t->structs[ip->alu.src.struct_id];
3644         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3645         src64 = *src64_ptr;
3646         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3647         src = src64 & src64_mask;
3648
3649         r = dst;
3650         r = ~r & 0xFFFF;
3651
3652         /* The first input (r) is a 16-bit number. The second and the third
3653          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3654          * three numbers (output r) is a 34-bit number.
3655          */
3656         r += (src >> 32) + (src & 0xFFFFFFFF);
3657
3658         /* The first input is a 16-bit number. The second input is an 18-bit
3659          * number. In the worst case scenario, the sum of the two numbers is a
3660          * 19-bit number.
3661          */
3662         r = (r & 0xFFFF) + (r >> 16);
3663
3664         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3665          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3666          */
3667         r = (r & 0xFFFF) + (r >> 16);
3668
3669         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3670          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3671          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3672          * therefore the output r is always a 16-bit number.
3673          */
3674         r = (r & 0xFFFF) + (r >> 16);
3675
3676         r = ~r & 0xFFFF;
3677         r = r ? r : 0xFFFF;
3678
3679         *dst16_ptr = (uint16_t)r;
3680
3681         /* Thread. */
3682         thread_ip_inc(p);
3683 }
3684
3685 static inline void
3686 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3687 {
3688         struct thread *t = &p->threads[p->thread_id];
3689         struct instruction *ip = t->ip;
3690         uint8_t *dst_struct, *src_struct;
3691         uint16_t *dst16_ptr, dst;
3692         uint64_t *src64_ptr, src64, src64_mask, src;
3693         uint64_t r;
3694
3695         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3696
3697         /* Structs. */
3698         dst_struct = t->structs[ip->alu.dst.struct_id];
3699         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3700         dst = *dst16_ptr;
3701
3702         src_struct = t->structs[ip->alu.src.struct_id];
3703         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3704         src64 = *src64_ptr;
3705         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3706         src = src64 & src64_mask;
3707
3708         r = dst;
3709         r = ~r & 0xFFFF;
3710
3711         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3712          * the following sequence of operations in 2's complement arithmetic:
3713          *    a '- b = (a - b) % 0xFFFF.
3714          *
3715          * In order to prevent an underflow for the below subtraction, in which
3716          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3717          * minuend), we first add a multiple of the 0xFFFF modulus to the
3718          * minuend. The number we add to the minuend needs to be a 34-bit number
3719          * or higher, so for readability reasons we picked the 36-bit multiple.
3720          * We are effectively turning the 16-bit minuend into a 36-bit number:
3721          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3722          */
3723         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3724
3725         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3726          * result (the output r) is a 36-bit number.
3727          */
3728         r -= (src >> 32) + (src & 0xFFFFFFFF);
3729
3730         /* The first input is a 16-bit number. The second input is a 20-bit
3731          * number. Their sum is a 21-bit number.
3732          */
3733         r = (r & 0xFFFF) + (r >> 16);
3734
3735         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3736          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3737          */
3738         r = (r & 0xFFFF) + (r >> 16);
3739
3740         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3741          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3742          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3743          * generated, therefore the output r is always a 16-bit number.
3744          */
3745         r = (r & 0xFFFF) + (r >> 16);
3746
3747         r = ~r & 0xFFFF;
3748         r = r ? r : 0xFFFF;
3749
3750         *dst16_ptr = (uint16_t)r;
3751
3752         /* Thread. */
3753         thread_ip_inc(p);
3754 }
3755
3756 static inline void
3757 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3758 {
3759         struct thread *t = &p->threads[p->thread_id];
3760         struct instruction *ip = t->ip;
3761         uint8_t *dst_struct, *src_struct;
3762         uint16_t *dst16_ptr;
3763         uint32_t *src32_ptr;
3764         uint64_t r0, r1;
3765
3766         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3767
3768         /* Structs. */
3769         dst_struct = t->structs[ip->alu.dst.struct_id];
3770         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3771
3772         src_struct = t->structs[ip->alu.src.struct_id];
3773         src32_ptr = (uint32_t *)&src_struct[0];
3774
3775         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3776         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3777         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3778         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3779         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3780
3781         /* The first input is a 16-bit number. The second input is a 19-bit
3782          * number. Their sum is a 20-bit number.
3783          */
3784         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3785
3786         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3787          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3788          */
3789         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3790
3791         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3792          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3793          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3794          * generated, therefore the output r is always a 16-bit number.
3795          */
3796         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3797
3798         r0 = ~r0 & 0xFFFF;
3799         r0 = r0 ? r0 : 0xFFFF;
3800
3801         *dst16_ptr = (uint16_t)r0;
3802
3803         /* Thread. */
3804         thread_ip_inc(p);
3805 }
3806
3807 static inline void
3808 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3809 {
3810         struct thread *t = &p->threads[p->thread_id];
3811         struct instruction *ip = t->ip;
3812         uint8_t *dst_struct, *src_struct;
3813         uint16_t *dst16_ptr;
3814         uint32_t *src32_ptr;
3815         uint64_t r = 0;
3816         uint32_t i;
3817
3818         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3819
3820         /* Structs. */
3821         dst_struct = t->structs[ip->alu.dst.struct_id];
3822         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3823
3824         src_struct = t->structs[ip->alu.src.struct_id];
3825         src32_ptr = (uint32_t *)&src_struct[0];
3826
3827         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3828          * Therefore, in the worst case scenario, a 35-bit number is added to a
3829          * 16-bit number (the input r), so the output r is 36-bit number.
3830          */
3831         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3832                 r += *src32_ptr;
3833
3834         /* The first input is a 16-bit number. The second input is a 20-bit
3835          * number. Their sum is a 21-bit number.
3836          */
3837         r = (r & 0xFFFF) + (r >> 16);
3838
3839         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3840          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3841          */
3842         r = (r & 0xFFFF) + (r >> 16);
3843
3844         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3845          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3846          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3847          * generated, therefore the output r is always a 16-bit number.
3848          */
3849         r = (r & 0xFFFF) + (r >> 16);
3850
3851         r = ~r & 0xFFFF;
3852         r = r ? r : 0xFFFF;
3853
3854         *dst16_ptr = (uint16_t)r;
3855
3856         /* Thread. */
3857         thread_ip_inc(p);
3858 }
3859
3860 /*
3861  * Register array.
3862  */
3863 static struct regarray *
3864 regarray_find(struct rte_swx_pipeline *p, const char *name);
3865
3866 static int
3867 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3868                       struct action *action,
3869                       char **tokens,
3870                       int n_tokens,
3871                       struct instruction *instr,
3872                       struct instruction_data *data __rte_unused)
3873 {
3874         char *regarray = tokens[1], *idx = tokens[2];
3875         struct regarray *r;
3876         struct field *fidx;
3877         uint32_t idx_struct_id, idx_val;
3878
3879         CHECK(n_tokens == 3, EINVAL);
3880
3881         r = regarray_find(p, regarray);
3882         CHECK(r, EINVAL);
3883
3884         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3885         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3886         if (fidx) {
3887                 CHECK(!fidx->var_size, EINVAL);
3888
3889                 instr->type = INSTR_REGPREFETCH_RM;
3890                 if (idx[0] == 'h')
3891                         instr->type = INSTR_REGPREFETCH_RH;
3892
3893                 instr->regarray.regarray_id = r->id;
3894                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3895                 instr->regarray.idx.n_bits = fidx->n_bits;
3896                 instr->regarray.idx.offset = fidx->offset / 8;
3897                 instr->regarray.dstsrc_val = 0; /* Unused. */
3898                 return 0;
3899         }
3900
3901         /* REGPREFETCH_RI. */
3902         idx_val = strtoul(idx, &idx, 0);
3903         CHECK(!idx[0], EINVAL);
3904
3905         instr->type = INSTR_REGPREFETCH_RI;
3906         instr->regarray.regarray_id = r->id;
3907         instr->regarray.idx_val = idx_val;
3908         instr->regarray.dstsrc_val = 0; /* Unused. */
3909         return 0;
3910 }
3911
3912 static int
3913 instr_regrd_translate(struct rte_swx_pipeline *p,
3914                       struct action *action,
3915                       char **tokens,
3916                       int n_tokens,
3917                       struct instruction *instr,
3918                       struct instruction_data *data __rte_unused)
3919 {
3920         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3921         struct regarray *r;
3922         struct field *fdst, *fidx;
3923         uint32_t dst_struct_id, idx_struct_id, idx_val;
3924
3925         CHECK(n_tokens == 4, EINVAL);
3926
3927         r = regarray_find(p, regarray);
3928         CHECK(r, EINVAL);
3929
3930         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3931         CHECK(fdst, EINVAL);
3932         CHECK(!fdst->var_size, EINVAL);
3933
3934         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3935         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3936         if (fidx) {
3937                 CHECK(!fidx->var_size, EINVAL);
3938
3939                 instr->type = INSTR_REGRD_MRM;
3940                 if (dst[0] == 'h' && idx[0] != 'h')
3941                         instr->type = INSTR_REGRD_HRM;
3942                 if (dst[0] != 'h' && idx[0] == 'h')
3943                         instr->type = INSTR_REGRD_MRH;
3944                 if (dst[0] == 'h' && idx[0] == 'h')
3945                         instr->type = INSTR_REGRD_HRH;
3946
3947                 instr->regarray.regarray_id = r->id;
3948                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3949                 instr->regarray.idx.n_bits = fidx->n_bits;
3950                 instr->regarray.idx.offset = fidx->offset / 8;
3951                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3952                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3953                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3954                 return 0;
3955         }
3956
3957         /* REGRD_MRI, REGRD_HRI. */
3958         idx_val = strtoul(idx, &idx, 0);
3959         CHECK(!idx[0], EINVAL);
3960
3961         instr->type = INSTR_REGRD_MRI;
3962         if (dst[0] == 'h')
3963                 instr->type = INSTR_REGRD_HRI;
3964
3965         instr->regarray.regarray_id = r->id;
3966         instr->regarray.idx_val = idx_val;
3967         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3968         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3969         instr->regarray.dstsrc.offset = fdst->offset / 8;
3970         return 0;
3971 }
3972
3973 static int
3974 instr_regwr_translate(struct rte_swx_pipeline *p,
3975                       struct action *action,
3976                       char **tokens,
3977                       int n_tokens,
3978                       struct instruction *instr,
3979                       struct instruction_data *data __rte_unused)
3980 {
3981         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3982         struct regarray *r;
3983         struct field *fidx, *fsrc;
3984         uint64_t src_val;
3985         uint32_t idx_struct_id, idx_val, src_struct_id;
3986
3987         CHECK(n_tokens == 4, EINVAL);
3988
3989         r = regarray_find(p, regarray);
3990         CHECK(r, EINVAL);
3991
3992         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3993         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3994         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3995         if (fidx && fsrc) {
3996                 CHECK(!fidx->var_size, EINVAL);
3997                 CHECK(!fsrc->var_size, EINVAL);
3998
3999                 instr->type = INSTR_REGWR_RMM;
4000                 if (idx[0] == 'h' && src[0] != 'h')
4001                         instr->type = INSTR_REGWR_RHM;
4002                 if (idx[0] != 'h' && src[0] == 'h')
4003                         instr->type = INSTR_REGWR_RMH;
4004                 if (idx[0] == 'h' && src[0] == 'h')
4005                         instr->type = INSTR_REGWR_RHH;
4006
4007                 instr->regarray.regarray_id = r->id;
4008                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4009                 instr->regarray.idx.n_bits = fidx->n_bits;
4010                 instr->regarray.idx.offset = fidx->offset / 8;
4011                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4012                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4013                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4014                 return 0;
4015         }
4016
4017         /* REGWR_RHI, REGWR_RMI. */
4018         if (fidx && !fsrc) {
4019                 CHECK(!fidx->var_size, EINVAL);
4020
4021                 src_val = strtoull(src, &src, 0);
4022                 CHECK(!src[0], EINVAL);
4023
4024                 instr->type = INSTR_REGWR_RMI;
4025                 if (idx[0] == 'h')
4026                         instr->type = INSTR_REGWR_RHI;
4027
4028                 instr->regarray.regarray_id = r->id;
4029                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4030                 instr->regarray.idx.n_bits = fidx->n_bits;
4031                 instr->regarray.idx.offset = fidx->offset / 8;
4032                 instr->regarray.dstsrc_val = src_val;
4033                 return 0;
4034         }
4035
4036         /* REGWR_RIH, REGWR_RIM. */
4037         if (!fidx && fsrc) {
4038                 idx_val = strtoul(idx, &idx, 0);
4039                 CHECK(!idx[0], EINVAL);
4040
4041                 CHECK(!fsrc->var_size, EINVAL);
4042
4043                 instr->type = INSTR_REGWR_RIM;
4044                 if (src[0] == 'h')
4045                         instr->type = INSTR_REGWR_RIH;
4046
4047                 instr->regarray.regarray_id = r->id;
4048                 instr->regarray.idx_val = idx_val;
4049                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4050                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4051                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4052                 return 0;
4053         }
4054
4055         /* REGWR_RII. */
4056         src_val = strtoull(src, &src, 0);
4057         CHECK(!src[0], EINVAL);
4058
4059         idx_val = strtoul(idx, &idx, 0);
4060         CHECK(!idx[0], EINVAL);
4061
4062         instr->type = INSTR_REGWR_RII;
4063         instr->regarray.idx_val = idx_val;
4064         instr->regarray.dstsrc_val = src_val;
4065
4066         return 0;
4067 }
4068
4069 static int
4070 instr_regadd_translate(struct rte_swx_pipeline *p,
4071                        struct action *action,
4072                        char **tokens,
4073                        int n_tokens,
4074                        struct instruction *instr,
4075                        struct instruction_data *data __rte_unused)
4076 {
4077         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4078         struct regarray *r;
4079         struct field *fidx, *fsrc;
4080         uint64_t src_val;
4081         uint32_t idx_struct_id, idx_val, src_struct_id;
4082
4083         CHECK(n_tokens == 4, EINVAL);
4084
4085         r = regarray_find(p, regarray);
4086         CHECK(r, EINVAL);
4087
4088         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4089         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4090         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4091         if (fidx && fsrc) {
4092                 CHECK(!fidx->var_size, EINVAL);
4093                 CHECK(!fsrc->var_size, EINVAL);
4094
4095                 instr->type = INSTR_REGADD_RMM;
4096                 if (idx[0] == 'h' && src[0] != 'h')
4097                         instr->type = INSTR_REGADD_RHM;
4098                 if (idx[0] != 'h' && src[0] == 'h')
4099                         instr->type = INSTR_REGADD_RMH;
4100                 if (idx[0] == 'h' && src[0] == 'h')
4101                         instr->type = INSTR_REGADD_RHH;
4102
4103                 instr->regarray.regarray_id = r->id;
4104                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4105                 instr->regarray.idx.n_bits = fidx->n_bits;
4106                 instr->regarray.idx.offset = fidx->offset / 8;
4107                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4108                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4109                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4110                 return 0;
4111         }
4112
4113         /* REGADD_RHI, REGADD_RMI. */
4114         if (fidx && !fsrc) {
4115                 CHECK(!fidx->var_size, EINVAL);
4116
4117                 src_val = strtoull(src, &src, 0);
4118                 CHECK(!src[0], EINVAL);
4119
4120                 instr->type = INSTR_REGADD_RMI;
4121                 if (idx[0] == 'h')
4122                         instr->type = INSTR_REGADD_RHI;
4123
4124                 instr->regarray.regarray_id = r->id;
4125                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4126                 instr->regarray.idx.n_bits = fidx->n_bits;
4127                 instr->regarray.idx.offset = fidx->offset / 8;
4128                 instr->regarray.dstsrc_val = src_val;
4129                 return 0;
4130         }
4131
4132         /* REGADD_RIH, REGADD_RIM. */
4133         if (!fidx && fsrc) {
4134                 idx_val = strtoul(idx, &idx, 0);
4135                 CHECK(!idx[0], EINVAL);
4136
4137                 CHECK(!fsrc->var_size, EINVAL);
4138
4139                 instr->type = INSTR_REGADD_RIM;
4140                 if (src[0] == 'h')
4141                         instr->type = INSTR_REGADD_RIH;
4142
4143                 instr->regarray.regarray_id = r->id;
4144                 instr->regarray.idx_val = idx_val;
4145                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4146                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4147                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4148                 return 0;
4149         }
4150
4151         /* REGADD_RII. */
4152         src_val = strtoull(src, &src, 0);
4153         CHECK(!src[0], EINVAL);
4154
4155         idx_val = strtoul(idx, &idx, 0);
4156         CHECK(!idx[0], EINVAL);
4157
4158         instr->type = INSTR_REGADD_RII;
4159         instr->regarray.idx_val = idx_val;
4160         instr->regarray.dstsrc_val = src_val;
4161         return 0;
4162 }
4163
4164 static inline uint64_t *
4165 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4166 {
4167         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4168         return r->regarray;
4169 }
4170
4171 static inline uint64_t
4172 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4173 {
4174         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4175
4176         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4177         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4178         uint64_t idx64 = *idx64_ptr;
4179         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4180         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4181
4182         return idx;
4183 }
4184
4185 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4186
4187 static inline uint64_t
4188 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4189 {
4190         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4191
4192         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4193         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4194         uint64_t idx64 = *idx64_ptr;
4195         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4196
4197         return idx;
4198 }
4199
4200 #else
4201
4202 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4203
4204 #endif
4205
4206 static inline uint64_t
4207 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4208 {
4209         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4210
4211         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4212
4213         return idx;
4214 }
4215
4216 static inline uint64_t
4217 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4218 {
4219         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4220         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4221         uint64_t src64 = *src64_ptr;
4222         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4223         uint64_t src = src64 & src64_mask;
4224
4225         return src;
4226 }
4227
4228 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4229
4230 static inline uint64_t
4231 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4232 {
4233         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4234         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4235         uint64_t src64 = *src64_ptr;
4236         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4237
4238         return src;
4239 }
4240
4241 #else
4242
4243 #define instr_regarray_src_nbo instr_regarray_src_hbo
4244
4245 #endif
4246
4247 static inline void
4248 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4249 {
4250         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4251         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4252         uint64_t dst64 = *dst64_ptr;
4253         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4254
4255         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4256
4257 }
4258
4259 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4260
4261 static inline void
4262 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4263 {
4264         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4265         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4266         uint64_t dst64 = *dst64_ptr;
4267         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4268
4269         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4270         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4271 }
4272
4273 #else
4274
4275 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4276
4277 #endif
4278
4279 static inline void
4280 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4281 {
4282         struct thread *t = &p->threads[p->thread_id];
4283         struct instruction *ip = t->ip;
4284         uint64_t *regarray, idx;
4285
4286         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4287
4288         /* Structs. */
4289         regarray = instr_regarray_regarray(p, ip);
4290         idx = instr_regarray_idx_nbo(p, t, ip);
4291         rte_prefetch0(&regarray[idx]);
4292
4293         /* Thread. */
4294         thread_ip_inc(p);
4295 }
4296
4297 static inline void
4298 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4299 {
4300         struct thread *t = &p->threads[p->thread_id];
4301         struct instruction *ip = t->ip;
4302         uint64_t *regarray, idx;
4303
4304         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4305
4306         /* Structs. */
4307         regarray = instr_regarray_regarray(p, ip);
4308         idx = instr_regarray_idx_hbo(p, t, ip);
4309         rte_prefetch0(&regarray[idx]);
4310
4311         /* Thread. */
4312         thread_ip_inc(p);
4313 }
4314
4315 static inline void
4316 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4317 {
4318         struct thread *t = &p->threads[p->thread_id];
4319         struct instruction *ip = t->ip;
4320         uint64_t *regarray, idx;
4321
4322         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4323
4324         /* Structs. */
4325         regarray = instr_regarray_regarray(p, ip);
4326         idx = instr_regarray_idx_imm(p, ip);
4327         rte_prefetch0(&regarray[idx]);
4328
4329         /* Thread. */
4330         thread_ip_inc(p);
4331 }
4332
4333 static inline void
4334 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4335 {
4336         struct thread *t = &p->threads[p->thread_id];
4337         struct instruction *ip = t->ip;
4338         uint64_t *regarray, idx;
4339
4340         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4341
4342         /* Structs. */
4343         regarray = instr_regarray_regarray(p, ip);
4344         idx = instr_regarray_idx_nbo(p, t, ip);
4345         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4346
4347         /* Thread. */
4348         thread_ip_inc(p);
4349 }
4350
4351 static inline void
4352 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4353 {
4354         struct thread *t = &p->threads[p->thread_id];
4355         struct instruction *ip = t->ip;
4356         uint64_t *regarray, idx;
4357
4358         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4359
4360         /* Structs. */
4361         regarray = instr_regarray_regarray(p, ip);
4362         idx = instr_regarray_idx_hbo(p, t, ip);
4363         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4364
4365         /* Thread. */
4366         thread_ip_inc(p);
4367 }
4368
4369 static inline void
4370 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4371 {
4372         struct thread *t = &p->threads[p->thread_id];
4373         struct instruction *ip = t->ip;
4374         uint64_t *regarray, idx;
4375
4376         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4377
4378         /* Structs. */
4379         regarray = instr_regarray_regarray(p, ip);
4380         idx = instr_regarray_idx_nbo(p, t, ip);
4381         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4382
4383         /* Thread. */
4384         thread_ip_inc(p);
4385 }
4386
4387 static inline void
4388 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4389 {
4390         struct thread *t = &p->threads[p->thread_id];
4391         struct instruction *ip = t->ip;
4392         uint64_t *regarray, idx;
4393
4394         /* Structs. */
4395         regarray = instr_regarray_regarray(p, ip);
4396         idx = instr_regarray_idx_hbo(p, t, ip);
4397         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4398
4399         /* Thread. */
4400         thread_ip_inc(p);
4401 }
4402
4403 static inline void
4404 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4405 {
4406         struct thread *t = &p->threads[p->thread_id];
4407         struct instruction *ip = t->ip;
4408         uint64_t *regarray, idx;
4409
4410         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4411
4412         /* Structs. */
4413         regarray = instr_regarray_regarray(p, ip);
4414         idx = instr_regarray_idx_imm(p, ip);
4415         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4416
4417         /* Thread. */
4418         thread_ip_inc(p);
4419 }
4420
4421 static inline void
4422 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4423 {
4424         struct thread *t = &p->threads[p->thread_id];
4425         struct instruction *ip = t->ip;
4426         uint64_t *regarray, idx;
4427
4428         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4429
4430         /* Structs. */
4431         regarray = instr_regarray_regarray(p, ip);
4432         idx = instr_regarray_idx_imm(p, ip);
4433         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4434
4435         /* Thread. */
4436         thread_ip_inc(p);
4437 }
4438
4439 static inline void
4440 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4441 {
4442         struct thread *t = &p->threads[p->thread_id];
4443         struct instruction *ip = t->ip;
4444         uint64_t *regarray, idx, src;
4445
4446         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4447
4448         /* Structs. */
4449         regarray = instr_regarray_regarray(p, ip);
4450         idx = instr_regarray_idx_nbo(p, t, ip);
4451         src = instr_regarray_src_nbo(t, ip);
4452         regarray[idx] = src;
4453
4454         /* Thread. */
4455         thread_ip_inc(p);
4456 }
4457
4458 static inline void
4459 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4460 {
4461         struct thread *t = &p->threads[p->thread_id];
4462         struct instruction *ip = t->ip;
4463         uint64_t *regarray, idx, src;
4464
4465         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4466
4467         /* Structs. */
4468         regarray = instr_regarray_regarray(p, ip);
4469         idx = instr_regarray_idx_nbo(p, t, ip);
4470         src = instr_regarray_src_hbo(t, ip);
4471         regarray[idx] = src;
4472
4473         /* Thread. */
4474         thread_ip_inc(p);
4475 }
4476
4477 static inline void
4478 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4479 {
4480         struct thread *t = &p->threads[p->thread_id];
4481         struct instruction *ip = t->ip;
4482         uint64_t *regarray, idx, src;
4483
4484         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4485
4486         /* Structs. */
4487         regarray = instr_regarray_regarray(p, ip);
4488         idx = instr_regarray_idx_hbo(p, t, ip);
4489         src = instr_regarray_src_nbo(t, ip);
4490         regarray[idx] = src;
4491
4492         /* Thread. */
4493         thread_ip_inc(p);
4494 }
4495
4496 static inline void
4497 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4498 {
4499         struct thread *t = &p->threads[p->thread_id];
4500         struct instruction *ip = t->ip;
4501         uint64_t *regarray, idx, src;
4502
4503         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4504
4505         /* Structs. */
4506         regarray = instr_regarray_regarray(p, ip);
4507         idx = instr_regarray_idx_hbo(p, t, ip);
4508         src = instr_regarray_src_hbo(t, ip);
4509         regarray[idx] = src;
4510
4511         /* Thread. */
4512         thread_ip_inc(p);
4513 }
4514
4515 static inline void
4516 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4517 {
4518         struct thread *t = &p->threads[p->thread_id];
4519         struct instruction *ip = t->ip;
4520         uint64_t *regarray, idx, src;
4521
4522         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4523
4524         /* Structs. */
4525         regarray = instr_regarray_regarray(p, ip);
4526         idx = instr_regarray_idx_nbo(p, t, ip);
4527         src = ip->regarray.dstsrc_val;
4528         regarray[idx] = src;
4529
4530         /* Thread. */
4531         thread_ip_inc(p);
4532 }
4533
4534 static inline void
4535 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4536 {
4537         struct thread *t = &p->threads[p->thread_id];
4538         struct instruction *ip = t->ip;
4539         uint64_t *regarray, idx, src;
4540
4541         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4542
4543         /* Structs. */
4544         regarray = instr_regarray_regarray(p, ip);
4545         idx = instr_regarray_idx_hbo(p, t, ip);
4546         src = ip->regarray.dstsrc_val;
4547         regarray[idx] = src;
4548
4549         /* Thread. */
4550         thread_ip_inc(p);
4551 }
4552
4553 static inline void
4554 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4555 {
4556         struct thread *t = &p->threads[p->thread_id];
4557         struct instruction *ip = t->ip;
4558         uint64_t *regarray, idx, src;
4559
4560         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4561
4562         /* Structs. */
4563         regarray = instr_regarray_regarray(p, ip);
4564         idx = instr_regarray_idx_imm(p, ip);
4565         src = instr_regarray_src_nbo(t, ip);
4566         regarray[idx] = src;
4567
4568         /* Thread. */
4569         thread_ip_inc(p);
4570 }
4571
4572 static inline void
4573 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4574 {
4575         struct thread *t = &p->threads[p->thread_id];
4576         struct instruction *ip = t->ip;
4577         uint64_t *regarray, idx, src;
4578
4579         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4580
4581         /* Structs. */
4582         regarray = instr_regarray_regarray(p, ip);
4583         idx = instr_regarray_idx_imm(p, ip);
4584         src = instr_regarray_src_hbo(t, ip);
4585         regarray[idx] = src;
4586
4587         /* Thread. */
4588         thread_ip_inc(p);
4589 }
4590
4591 static inline void
4592 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4593 {
4594         struct thread *t = &p->threads[p->thread_id];
4595         struct instruction *ip = t->ip;
4596         uint64_t *regarray, idx, src;
4597
4598         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4599
4600         /* Structs. */
4601         regarray = instr_regarray_regarray(p, ip);
4602         idx = instr_regarray_idx_imm(p, ip);
4603         src = ip->regarray.dstsrc_val;
4604         regarray[idx] = src;
4605
4606         /* Thread. */
4607         thread_ip_inc(p);
4608 }
4609
4610 static inline void
4611 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4612 {
4613         struct thread *t = &p->threads[p->thread_id];
4614         struct instruction *ip = t->ip;
4615         uint64_t *regarray, idx, src;
4616
4617         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4618
4619         /* Structs. */
4620         regarray = instr_regarray_regarray(p, ip);
4621         idx = instr_regarray_idx_nbo(p, t, ip);
4622         src = instr_regarray_src_nbo(t, ip);
4623         regarray[idx] += src;
4624
4625         /* Thread. */
4626         thread_ip_inc(p);
4627 }
4628
4629 static inline void
4630 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4631 {
4632         struct thread *t = &p->threads[p->thread_id];
4633         struct instruction *ip = t->ip;
4634         uint64_t *regarray, idx, src;
4635
4636         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4637
4638         /* Structs. */
4639         regarray = instr_regarray_regarray(p, ip);
4640         idx = instr_regarray_idx_nbo(p, t, ip);
4641         src = instr_regarray_src_hbo(t, ip);
4642         regarray[idx] += src;
4643
4644         /* Thread. */
4645         thread_ip_inc(p);
4646 }
4647
4648 static inline void
4649 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4650 {
4651         struct thread *t = &p->threads[p->thread_id];
4652         struct instruction *ip = t->ip;
4653         uint64_t *regarray, idx, src;
4654
4655         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4656
4657         /* Structs. */
4658         regarray = instr_regarray_regarray(p, ip);
4659         idx = instr_regarray_idx_hbo(p, t, ip);
4660         src = instr_regarray_src_nbo(t, ip);
4661         regarray[idx] += src;
4662
4663         /* Thread. */
4664         thread_ip_inc(p);
4665 }
4666
4667 static inline void
4668 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4669 {
4670         struct thread *t = &p->threads[p->thread_id];
4671         struct instruction *ip = t->ip;
4672         uint64_t *regarray, idx, src;
4673
4674         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4675
4676         /* Structs. */
4677         regarray = instr_regarray_regarray(p, ip);
4678         idx = instr_regarray_idx_hbo(p, t, ip);
4679         src = instr_regarray_src_hbo(t, ip);
4680         regarray[idx] += src;
4681
4682         /* Thread. */
4683         thread_ip_inc(p);
4684 }
4685
4686 static inline void
4687 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4688 {
4689         struct thread *t = &p->threads[p->thread_id];
4690         struct instruction *ip = t->ip;
4691         uint64_t *regarray, idx, src;
4692
4693         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4694
4695         /* Structs. */
4696         regarray = instr_regarray_regarray(p, ip);
4697         idx = instr_regarray_idx_nbo(p, t, ip);
4698         src = ip->regarray.dstsrc_val;
4699         regarray[idx] += src;
4700
4701         /* Thread. */
4702         thread_ip_inc(p);
4703 }
4704
4705 static inline void
4706 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4707 {
4708         struct thread *t = &p->threads[p->thread_id];
4709         struct instruction *ip = t->ip;
4710         uint64_t *regarray, idx, src;
4711
4712         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4713
4714         /* Structs. */
4715         regarray = instr_regarray_regarray(p, ip);
4716         idx = instr_regarray_idx_hbo(p, t, ip);
4717         src = ip->regarray.dstsrc_val;
4718         regarray[idx] += src;
4719
4720         /* Thread. */
4721         thread_ip_inc(p);
4722 }
4723
4724 static inline void
4725 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4726 {
4727         struct thread *t = &p->threads[p->thread_id];
4728         struct instruction *ip = t->ip;
4729         uint64_t *regarray, idx, src;
4730
4731         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4732
4733         /* Structs. */
4734         regarray = instr_regarray_regarray(p, ip);
4735         idx = instr_regarray_idx_imm(p, ip);
4736         src = instr_regarray_src_nbo(t, ip);
4737         regarray[idx] += src;
4738
4739         /* Thread. */
4740         thread_ip_inc(p);
4741 }
4742
4743 static inline void
4744 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4745 {
4746         struct thread *t = &p->threads[p->thread_id];
4747         struct instruction *ip = t->ip;
4748         uint64_t *regarray, idx, src;
4749
4750         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4751
4752         /* Structs. */
4753         regarray = instr_regarray_regarray(p, ip);
4754         idx = instr_regarray_idx_imm(p, ip);
4755         src = instr_regarray_src_hbo(t, ip);
4756         regarray[idx] += src;
4757
4758         /* Thread. */
4759         thread_ip_inc(p);
4760 }
4761
4762 static inline void
4763 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4764 {
4765         struct thread *t = &p->threads[p->thread_id];
4766         struct instruction *ip = t->ip;
4767         uint64_t *regarray, idx, src;
4768
4769         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4770
4771         /* Structs. */
4772         regarray = instr_regarray_regarray(p, ip);
4773         idx = instr_regarray_idx_imm(p, ip);
4774         src = ip->regarray.dstsrc_val;
4775         regarray[idx] += src;
4776
4777         /* Thread. */
4778         thread_ip_inc(p);
4779 }
4780
4781 /*
4782  * metarray.
4783  */
4784 static struct metarray *
4785 metarray_find(struct rte_swx_pipeline *p, const char *name);
4786
4787 static int
4788 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4789                             struct action *action,
4790                             char **tokens,
4791                             int n_tokens,
4792                             struct instruction *instr,
4793                             struct instruction_data *data __rte_unused)
4794 {
4795         char *metarray = tokens[1], *idx = tokens[2];
4796         struct metarray *m;
4797         struct field *fidx;
4798         uint32_t idx_struct_id, idx_val;
4799
4800         CHECK(n_tokens == 3, EINVAL);
4801
4802         m = metarray_find(p, metarray);
4803         CHECK(m, EINVAL);
4804
4805         /* METPREFETCH_H, METPREFETCH_M. */
4806         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4807         if (fidx) {
4808                 CHECK(!fidx->var_size, EINVAL);
4809
4810                 instr->type = INSTR_METPREFETCH_M;
4811                 if (idx[0] == 'h')
4812                         instr->type = INSTR_METPREFETCH_H;
4813
4814                 instr->meter.metarray_id = m->id;
4815                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4816                 instr->meter.idx.n_bits = fidx->n_bits;
4817                 instr->meter.idx.offset = fidx->offset / 8;
4818                 return 0;
4819         }
4820
4821         /* METPREFETCH_I. */
4822         idx_val = strtoul(idx, &idx, 0);
4823         CHECK(!idx[0], EINVAL);
4824
4825         instr->type = INSTR_METPREFETCH_I;
4826         instr->meter.metarray_id = m->id;
4827         instr->meter.idx_val = idx_val;
4828         return 0;
4829 }
4830
4831 static int
4832 instr_meter_translate(struct rte_swx_pipeline *p,
4833                       struct action *action,
4834                       char **tokens,
4835                       int n_tokens,
4836                       struct instruction *instr,
4837                       struct instruction_data *data __rte_unused)
4838 {
4839         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4840         char *color_in = tokens[4], *color_out = tokens[5];
4841         struct metarray *m;
4842         struct field *fidx, *flength, *fcin, *fcout;
4843         uint32_t idx_struct_id, length_struct_id;
4844         uint32_t color_in_struct_id, color_out_struct_id;
4845
4846         CHECK(n_tokens == 6, EINVAL);
4847
4848         m = metarray_find(p, metarray);
4849         CHECK(m, EINVAL);
4850
4851         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4852
4853         flength = struct_field_parse(p, action, length, &length_struct_id);
4854         CHECK(flength, EINVAL);
4855         CHECK(!flength->var_size, EINVAL);
4856
4857         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4858
4859         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4860         CHECK(fcout, EINVAL);
4861         CHECK(!fcout->var_size, EINVAL);
4862
4863         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4864         if (fidx && fcin) {
4865                 CHECK(!fidx->var_size, EINVAL);
4866                 CHECK(!fcin->var_size, EINVAL);
4867
4868                 instr->type = INSTR_METER_MMM;
4869                 if (idx[0] == 'h' && length[0] == 'h')
4870                         instr->type = INSTR_METER_HHM;
4871                 if (idx[0] == 'h' && length[0] != 'h')
4872                         instr->type = INSTR_METER_HMM;
4873                 if (idx[0] != 'h' && length[0] == 'h')
4874                         instr->type = INSTR_METER_MHM;
4875
4876                 instr->meter.metarray_id = m->id;
4877
4878                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4879                 instr->meter.idx.n_bits = fidx->n_bits;
4880                 instr->meter.idx.offset = fidx->offset / 8;
4881
4882                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4883                 instr->meter.length.n_bits = flength->n_bits;
4884                 instr->meter.length.offset = flength->offset / 8;
4885
4886                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4887                 instr->meter.color_in.n_bits = fcin->n_bits;
4888                 instr->meter.color_in.offset = fcin->offset / 8;
4889
4890                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4891                 instr->meter.color_out.n_bits = fcout->n_bits;
4892                 instr->meter.color_out.offset = fcout->offset / 8;
4893
4894                 return 0;
4895         }
4896
4897         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4898         if (fidx && !fcin) {
4899                 uint32_t color_in_val;
4900
4901                 CHECK(!fidx->var_size, EINVAL);
4902
4903                 color_in_val = strtoul(color_in, &color_in, 0);
4904                 CHECK(!color_in[0], EINVAL);
4905
4906                 instr->type = INSTR_METER_MMI;
4907                 if (idx[0] == 'h' && length[0] == 'h')
4908                         instr->type = INSTR_METER_HHI;
4909                 if (idx[0] == 'h' && length[0] != 'h')
4910                         instr->type = INSTR_METER_HMI;
4911                 if (idx[0] != 'h' && length[0] == 'h')
4912                         instr->type = INSTR_METER_MHI;
4913
4914                 instr->meter.metarray_id = m->id;
4915
4916                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4917                 instr->meter.idx.n_bits = fidx->n_bits;
4918                 instr->meter.idx.offset = fidx->offset / 8;
4919
4920                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4921                 instr->meter.length.n_bits = flength->n_bits;
4922                 instr->meter.length.offset = flength->offset / 8;
4923
4924                 instr->meter.color_in_val = color_in_val;
4925
4926                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4927                 instr->meter.color_out.n_bits = fcout->n_bits;
4928                 instr->meter.color_out.offset = fcout->offset / 8;
4929
4930                 return 0;
4931         }
4932
4933         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4934         if (!fidx && fcin) {
4935                 uint32_t idx_val;
4936
4937                 idx_val = strtoul(idx, &idx, 0);
4938                 CHECK(!idx[0], EINVAL);
4939
4940                 CHECK(!fcin->var_size, EINVAL);
4941
4942                 instr->type = INSTR_METER_IMM;
4943                 if (length[0] == 'h')
4944                         instr->type = INSTR_METER_IHM;
4945
4946                 instr->meter.metarray_id = m->id;
4947
4948                 instr->meter.idx_val = idx_val;
4949
4950                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4951                 instr->meter.length.n_bits = flength->n_bits;
4952                 instr->meter.length.offset = flength->offset / 8;
4953
4954                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4955                 instr->meter.color_in.n_bits = fcin->n_bits;
4956                 instr->meter.color_in.offset = fcin->offset / 8;
4957
4958                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4959                 instr->meter.color_out.n_bits = fcout->n_bits;
4960                 instr->meter.color_out.offset = fcout->offset / 8;
4961
4962                 return 0;
4963         }
4964
4965         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4966         if (!fidx && !fcin) {
4967                 uint32_t idx_val, color_in_val;
4968
4969                 idx_val = strtoul(idx, &idx, 0);
4970                 CHECK(!idx[0], EINVAL);
4971
4972                 color_in_val = strtoul(color_in, &color_in, 0);
4973                 CHECK(!color_in[0], EINVAL);
4974
4975                 instr->type = INSTR_METER_IMI;
4976                 if (length[0] == 'h')
4977                         instr->type = INSTR_METER_IHI;
4978
4979                 instr->meter.metarray_id = m->id;
4980
4981                 instr->meter.idx_val = idx_val;
4982
4983                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4984                 instr->meter.length.n_bits = flength->n_bits;
4985                 instr->meter.length.offset = flength->offset / 8;
4986
4987                 instr->meter.color_in_val = color_in_val;
4988
4989                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4990                 instr->meter.color_out.n_bits = fcout->n_bits;
4991                 instr->meter.color_out.offset = fcout->offset / 8;
4992
4993                 return 0;
4994         }
4995
4996         CHECK(0, EINVAL);
4997 }
4998
4999 static inline struct meter *
5000 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5001 {
5002         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5003
5004         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5005         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5006         uint64_t idx64 = *idx64_ptr;
5007         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5008         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5009
5010         return &r->metarray[idx];
5011 }
5012
5013 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5014
5015 static inline struct meter *
5016 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5017 {
5018         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5019
5020         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5021         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5022         uint64_t idx64 = *idx64_ptr;
5023         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5024
5025         return &r->metarray[idx];
5026 }
5027
5028 #else
5029
5030 #define instr_meter_idx_nbo instr_meter_idx_hbo
5031
5032 #endif
5033
5034 static inline struct meter *
5035 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5036 {
5037         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5038
5039         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5040
5041         return &r->metarray[idx];
5042 }
5043
5044 static inline uint32_t
5045 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5046 {
5047         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5048         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5049         uint64_t src64 = *src64_ptr;
5050         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5051         uint64_t src = src64 & src64_mask;
5052
5053         return (uint32_t)src;
5054 }
5055
5056 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5057
5058 static inline uint32_t
5059 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5060 {
5061         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5062         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5063         uint64_t src64 = *src64_ptr;
5064         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5065
5066         return (uint32_t)src;
5067 }
5068
5069 #else
5070
5071 #define instr_meter_length_nbo instr_meter_length_hbo
5072
5073 #endif
5074
5075 static inline enum rte_color
5076 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5077 {
5078         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5079         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5080         uint64_t src64 = *src64_ptr;
5081         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5082         uint64_t src = src64 & src64_mask;
5083
5084         return (enum rte_color)src;
5085 }
5086
5087 static inline void
5088 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5089 {
5090         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5091         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5092         uint64_t dst64 = *dst64_ptr;
5093         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5094
5095         uint64_t src = (uint64_t)color_out;
5096
5097         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5098 }
5099
5100 static inline void
5101 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5102 {
5103         struct thread *t = &p->threads[p->thread_id];
5104         struct instruction *ip = t->ip;
5105         struct meter *m;
5106
5107         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5108
5109         /* Structs. */
5110         m = instr_meter_idx_nbo(p, t, ip);
5111         rte_prefetch0(m);
5112
5113         /* Thread. */
5114         thread_ip_inc(p);
5115 }
5116
5117 static inline void
5118 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5119 {
5120         struct thread *t = &p->threads[p->thread_id];
5121         struct instruction *ip = t->ip;
5122         struct meter *m;
5123
5124         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5125
5126         /* Structs. */
5127         m = instr_meter_idx_hbo(p, t, ip);
5128         rte_prefetch0(m);
5129
5130         /* Thread. */
5131         thread_ip_inc(p);
5132 }
5133
5134 static inline void
5135 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5136 {
5137         struct thread *t = &p->threads[p->thread_id];
5138         struct instruction *ip = t->ip;
5139         struct meter *m;
5140
5141         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5142
5143         /* Structs. */
5144         m = instr_meter_idx_imm(p, ip);
5145         rte_prefetch0(m);
5146
5147         /* Thread. */
5148         thread_ip_inc(p);
5149 }
5150
5151 static inline void
5152 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5153 {
5154         struct thread *t = &p->threads[p->thread_id];
5155         struct instruction *ip = t->ip;
5156         struct meter *m;
5157         uint64_t time, n_pkts, n_bytes;
5158         uint32_t length;
5159         enum rte_color color_in, color_out;
5160
5161         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5162
5163         /* Structs. */
5164         m = instr_meter_idx_nbo(p, t, ip);
5165         rte_prefetch0(m->n_pkts);
5166         time = rte_get_tsc_cycles();
5167         length = instr_meter_length_nbo(t, ip);
5168         color_in = instr_meter_color_in_hbo(t, ip);
5169
5170         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5171                 &m->profile->profile,
5172                 time,
5173                 length,
5174                 color_in);
5175
5176         color_out &= m->color_mask;
5177
5178         n_pkts = m->n_pkts[color_out];
5179         n_bytes = m->n_bytes[color_out];
5180
5181         instr_meter_color_out_hbo_set(t, ip, color_out);
5182
5183         m->n_pkts[color_out] = n_pkts + 1;
5184         m->n_bytes[color_out] = n_bytes + length;
5185
5186         /* Thread. */
5187         thread_ip_inc(p);
5188 }
5189
5190 static inline void
5191 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5192 {
5193         struct thread *t = &p->threads[p->thread_id];
5194         struct instruction *ip = t->ip;
5195         struct meter *m;
5196         uint64_t time, n_pkts, n_bytes;
5197         uint32_t length;
5198         enum rte_color color_in, color_out;
5199
5200         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5201
5202         /* Structs. */
5203         m = instr_meter_idx_nbo(p, t, ip);
5204         rte_prefetch0(m->n_pkts);
5205         time = rte_get_tsc_cycles();
5206         length = instr_meter_length_nbo(t, ip);
5207         color_in = (enum rte_color)ip->meter.color_in_val;
5208
5209         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5210                 &m->profile->profile,
5211                 time,
5212                 length,
5213                 color_in);
5214
5215         color_out &= m->color_mask;
5216
5217         n_pkts = m->n_pkts[color_out];
5218         n_bytes = m->n_bytes[color_out];
5219
5220         instr_meter_color_out_hbo_set(t, ip, color_out);
5221
5222         m->n_pkts[color_out] = n_pkts + 1;
5223         m->n_bytes[color_out] = n_bytes + length;
5224
5225         /* Thread. */
5226         thread_ip_inc(p);
5227 }
5228
5229 static inline void
5230 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5231 {
5232         struct thread *t = &p->threads[p->thread_id];
5233         struct instruction *ip = t->ip;
5234         struct meter *m;
5235         uint64_t time, n_pkts, n_bytes;
5236         uint32_t length;
5237         enum rte_color color_in, color_out;
5238
5239         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5240
5241         /* Structs. */
5242         m = instr_meter_idx_nbo(p, t, ip);
5243         rte_prefetch0(m->n_pkts);
5244         time = rte_get_tsc_cycles();
5245         length = instr_meter_length_hbo(t, ip);
5246         color_in = instr_meter_color_in_hbo(t, ip);
5247
5248         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5249                 &m->profile->profile,
5250                 time,
5251                 length,
5252                 color_in);
5253
5254         color_out &= m->color_mask;
5255
5256         n_pkts = m->n_pkts[color_out];
5257         n_bytes = m->n_bytes[color_out];
5258
5259         instr_meter_color_out_hbo_set(t, ip, color_out);
5260
5261         m->n_pkts[color_out] = n_pkts + 1;
5262         m->n_bytes[color_out] = n_bytes + length;
5263
5264         /* Thread. */
5265         thread_ip_inc(p);
5266 }
5267 static inline void
5268 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5269 {
5270         struct thread *t = &p->threads[p->thread_id];
5271         struct instruction *ip = t->ip;
5272         struct meter *m;
5273         uint64_t time, n_pkts, n_bytes;
5274         uint32_t length;
5275         enum rte_color color_in, color_out;
5276
5277         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5278
5279         /* Structs. */
5280         m = instr_meter_idx_nbo(p, t, ip);
5281         rte_prefetch0(m->n_pkts);
5282         time = rte_get_tsc_cycles();
5283         length = instr_meter_length_hbo(t, ip);
5284         color_in = (enum rte_color)ip->meter.color_in_val;
5285
5286         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5287                 &m->profile->profile,
5288                 time,
5289                 length,
5290                 color_in);
5291
5292         color_out &= m->color_mask;
5293
5294         n_pkts = m->n_pkts[color_out];
5295         n_bytes = m->n_bytes[color_out];
5296
5297         instr_meter_color_out_hbo_set(t, ip, color_out);
5298
5299         m->n_pkts[color_out] = n_pkts + 1;
5300         m->n_bytes[color_out] = n_bytes + length;
5301
5302         /* Thread. */
5303         thread_ip_inc(p);
5304 }
5305
5306 static inline void
5307 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5308 {
5309         struct thread *t = &p->threads[p->thread_id];
5310         struct instruction *ip = t->ip;
5311         struct meter *m;
5312         uint64_t time, n_pkts, n_bytes;
5313         uint32_t length;
5314         enum rte_color color_in, color_out;
5315
5316         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5317
5318         /* Structs. */
5319         m = instr_meter_idx_hbo(p, t, ip);
5320         rte_prefetch0(m->n_pkts);
5321         time = rte_get_tsc_cycles();
5322         length = instr_meter_length_nbo(t, ip);
5323         color_in = instr_meter_color_in_hbo(t, ip);
5324
5325         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5326                 &m->profile->profile,
5327                 time,
5328                 length,
5329                 color_in);
5330
5331         color_out &= m->color_mask;
5332
5333         n_pkts = m->n_pkts[color_out];
5334         n_bytes = m->n_bytes[color_out];
5335
5336         instr_meter_color_out_hbo_set(t, ip, color_out);
5337
5338         m->n_pkts[color_out] = n_pkts + 1;
5339         m->n_bytes[color_out] = n_bytes + length;
5340
5341         /* Thread. */
5342         thread_ip_inc(p);
5343 }
5344
5345 static inline void
5346 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5347 {
5348         struct thread *t = &p->threads[p->thread_id];
5349         struct instruction *ip = t->ip;
5350         struct meter *m;
5351         uint64_t time, n_pkts, n_bytes;
5352         uint32_t length;
5353         enum rte_color color_in, color_out;
5354
5355         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5356
5357         /* Structs. */
5358         m = instr_meter_idx_hbo(p, t, ip);
5359         rte_prefetch0(m->n_pkts);
5360         time = rte_get_tsc_cycles();
5361         length = instr_meter_length_nbo(t, ip);
5362         color_in = (enum rte_color)ip->meter.color_in_val;
5363
5364         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5365                 &m->profile->profile,
5366                 time,
5367                 length,
5368                 color_in);
5369
5370         color_out &= m->color_mask;
5371
5372         n_pkts = m->n_pkts[color_out];
5373         n_bytes = m->n_bytes[color_out];
5374
5375         instr_meter_color_out_hbo_set(t, ip, color_out);
5376
5377         m->n_pkts[color_out] = n_pkts + 1;
5378         m->n_bytes[color_out] = n_bytes + length;
5379
5380         /* Thread. */
5381         thread_ip_inc(p);
5382 }
5383
5384 static inline void
5385 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5386 {
5387         struct thread *t = &p->threads[p->thread_id];
5388         struct instruction *ip = t->ip;
5389         struct meter *m;
5390         uint64_t time, n_pkts, n_bytes;
5391         uint32_t length;
5392         enum rte_color color_in, color_out;
5393
5394         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5395
5396         /* Structs. */
5397         m = instr_meter_idx_hbo(p, t, ip);
5398         rte_prefetch0(m->n_pkts);
5399         time = rte_get_tsc_cycles();
5400         length = instr_meter_length_hbo(t, ip);
5401         color_in = instr_meter_color_in_hbo(t, ip);
5402
5403         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5404                 &m->profile->profile,
5405                 time,
5406                 length,
5407                 color_in);
5408
5409         color_out &= m->color_mask;
5410
5411         n_pkts = m->n_pkts[color_out];
5412         n_bytes = m->n_bytes[color_out];
5413
5414         instr_meter_color_out_hbo_set(t, ip, color_out);
5415
5416         m->n_pkts[color_out] = n_pkts + 1;
5417         m->n_bytes[color_out] = n_bytes + length;
5418
5419         /* Thread. */
5420         thread_ip_inc(p);
5421 }
5422
5423 static inline void
5424 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5425 {
5426         struct thread *t = &p->threads[p->thread_id];
5427         struct instruction *ip = t->ip;
5428         struct meter *m;
5429         uint64_t time, n_pkts, n_bytes;
5430         uint32_t length;
5431         enum rte_color color_in, color_out;
5432
5433         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5434
5435         /* Structs. */
5436         m = instr_meter_idx_hbo(p, t, ip);
5437         rte_prefetch0(m->n_pkts);
5438         time = rte_get_tsc_cycles();
5439         length = instr_meter_length_hbo(t, ip);
5440         color_in = (enum rte_color)ip->meter.color_in_val;
5441
5442         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5443                 &m->profile->profile,
5444                 time,
5445                 length,
5446                 color_in);
5447
5448         color_out &= m->color_mask;
5449
5450         n_pkts = m->n_pkts[color_out];
5451         n_bytes = m->n_bytes[color_out];
5452
5453         instr_meter_color_out_hbo_set(t, ip, color_out);
5454
5455         m->n_pkts[color_out] = n_pkts + 1;
5456         m->n_bytes[color_out] = n_bytes + length;
5457
5458         /* Thread. */
5459         thread_ip_inc(p);
5460 }
5461
5462 static inline void
5463 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5464 {
5465         struct thread *t = &p->threads[p->thread_id];
5466         struct instruction *ip = t->ip;
5467         struct meter *m;
5468         uint64_t time, n_pkts, n_bytes;
5469         uint32_t length;
5470         enum rte_color color_in, color_out;
5471
5472         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5473
5474         /* Structs. */
5475         m = instr_meter_idx_imm(p, ip);
5476         rte_prefetch0(m->n_pkts);
5477         time = rte_get_tsc_cycles();
5478         length = instr_meter_length_nbo(t, ip);
5479         color_in = instr_meter_color_in_hbo(t, ip);
5480
5481         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5482                 &m->profile->profile,
5483                 time,
5484                 length,
5485                 color_in);
5486
5487         color_out &= m->color_mask;
5488
5489         n_pkts = m->n_pkts[color_out];
5490         n_bytes = m->n_bytes[color_out];
5491
5492         instr_meter_color_out_hbo_set(t, ip, color_out);
5493
5494         m->n_pkts[color_out] = n_pkts + 1;
5495         m->n_bytes[color_out] = n_bytes + length;
5496
5497         /* Thread. */
5498         thread_ip_inc(p);
5499 }
5500
5501 static inline void
5502 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5503 {
5504         struct thread *t = &p->threads[p->thread_id];
5505         struct instruction *ip = t->ip;
5506         struct meter *m;
5507         uint64_t time, n_pkts, n_bytes;
5508         uint32_t length;
5509         enum rte_color color_in, color_out;
5510
5511         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5512
5513         /* Structs. */
5514         m = instr_meter_idx_imm(p, ip);
5515         rte_prefetch0(m->n_pkts);
5516         time = rte_get_tsc_cycles();
5517         length = instr_meter_length_nbo(t, ip);
5518         color_in = (enum rte_color)ip->meter.color_in_val;
5519
5520         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5521                 &m->profile->profile,
5522                 time,
5523                 length,
5524                 color_in);
5525
5526         color_out &= m->color_mask;
5527
5528         n_pkts = m->n_pkts[color_out];
5529         n_bytes = m->n_bytes[color_out];
5530
5531         instr_meter_color_out_hbo_set(t, ip, color_out);
5532
5533         m->n_pkts[color_out] = n_pkts + 1;
5534         m->n_bytes[color_out] = n_bytes + length;
5535
5536         /* Thread. */
5537         thread_ip_inc(p);
5538 }
5539
5540 static inline void
5541 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5542 {
5543         struct thread *t = &p->threads[p->thread_id];
5544         struct instruction *ip = t->ip;
5545         struct meter *m;
5546         uint64_t time, n_pkts, n_bytes;
5547         uint32_t length;
5548         enum rte_color color_in, color_out;
5549
5550         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5551
5552         /* Structs. */
5553         m = instr_meter_idx_imm(p, ip);
5554         rte_prefetch0(m->n_pkts);
5555         time = rte_get_tsc_cycles();
5556         length = instr_meter_length_hbo(t, ip);
5557         color_in = instr_meter_color_in_hbo(t, ip);
5558
5559         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5560                 &m->profile->profile,
5561                 time,
5562                 length,
5563                 color_in);
5564
5565         color_out &= m->color_mask;
5566
5567         n_pkts = m->n_pkts[color_out];
5568         n_bytes = m->n_bytes[color_out];
5569
5570         instr_meter_color_out_hbo_set(t, ip, color_out);
5571
5572         m->n_pkts[color_out] = n_pkts + 1;
5573         m->n_bytes[color_out] = n_bytes + length;
5574
5575         /* Thread. */
5576         thread_ip_inc(p);
5577 }
5578 static inline void
5579 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5580 {
5581         struct thread *t = &p->threads[p->thread_id];
5582         struct instruction *ip = t->ip;
5583         struct meter *m;
5584         uint64_t time, n_pkts, n_bytes;
5585         uint32_t length;
5586         enum rte_color color_in, color_out;
5587
5588         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5589
5590         /* Structs. */
5591         m = instr_meter_idx_imm(p, ip);
5592         rte_prefetch0(m->n_pkts);
5593         time = rte_get_tsc_cycles();
5594         length = instr_meter_length_hbo(t, ip);
5595         color_in = (enum rte_color)ip->meter.color_in_val;
5596
5597         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5598                 &m->profile->profile,
5599                 time,
5600                 length,
5601                 color_in);
5602
5603         color_out &= m->color_mask;
5604
5605         n_pkts = m->n_pkts[color_out];
5606         n_bytes = m->n_bytes[color_out];
5607
5608         instr_meter_color_out_hbo_set(t, ip, color_out);
5609
5610         m->n_pkts[color_out] = n_pkts + 1;
5611         m->n_bytes[color_out] = n_bytes + length;
5612
5613         /* Thread. */
5614         thread_ip_inc(p);
5615 }
5616
5617 /*
5618  * jmp.
5619  */
5620 static int
5621 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5622                     struct action *action __rte_unused,
5623                     char **tokens,
5624                     int n_tokens,
5625                     struct instruction *instr,
5626                     struct instruction_data *data)
5627 {
5628         CHECK(n_tokens == 2, EINVAL);
5629
5630         strcpy(data->jmp_label, tokens[1]);
5631
5632         instr->type = INSTR_JMP;
5633         instr->jmp.ip = NULL; /* Resolved later. */
5634         return 0;
5635 }
5636
5637 static int
5638 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5639                           struct action *action __rte_unused,
5640                           char **tokens,
5641                           int n_tokens,
5642                           struct instruction *instr,
5643                           struct instruction_data *data)
5644 {
5645         struct header *h;
5646
5647         CHECK(n_tokens == 3, EINVAL);
5648
5649         strcpy(data->jmp_label, tokens[1]);
5650
5651         h = header_parse(p, tokens[2]);
5652         CHECK(h, EINVAL);
5653
5654         instr->type = INSTR_JMP_VALID;
5655         instr->jmp.ip = NULL; /* Resolved later. */
5656         instr->jmp.header_id = h->id;
5657         return 0;
5658 }
5659
5660 static int
5661 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5662                             struct action *action __rte_unused,
5663                             char **tokens,
5664                             int n_tokens,
5665                             struct instruction *instr,
5666                             struct instruction_data *data)
5667 {
5668         struct header *h;
5669
5670         CHECK(n_tokens == 3, EINVAL);
5671
5672         strcpy(data->jmp_label, tokens[1]);
5673
5674         h = header_parse(p, tokens[2]);
5675         CHECK(h, EINVAL);
5676
5677         instr->type = INSTR_JMP_INVALID;
5678         instr->jmp.ip = NULL; /* Resolved later. */
5679         instr->jmp.header_id = h->id;
5680         return 0;
5681 }
5682
5683 static int
5684 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5685                         struct action *action,
5686                         char **tokens,
5687                         int n_tokens,
5688                         struct instruction *instr,
5689                         struct instruction_data *data)
5690 {
5691         CHECK(!action, EINVAL);
5692         CHECK(n_tokens == 2, EINVAL);
5693
5694         strcpy(data->jmp_label, tokens[1]);
5695
5696         instr->type = INSTR_JMP_HIT;
5697         instr->jmp.ip = NULL; /* Resolved later. */
5698         return 0;
5699 }
5700
5701 static int
5702 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5703                          struct action *action,
5704                          char **tokens,
5705                          int n_tokens,
5706                          struct instruction *instr,
5707                          struct instruction_data *data)
5708 {
5709         CHECK(!action, EINVAL);
5710         CHECK(n_tokens == 2, EINVAL);
5711
5712         strcpy(data->jmp_label, tokens[1]);
5713
5714         instr->type = INSTR_JMP_MISS;
5715         instr->jmp.ip = NULL; /* Resolved later. */
5716         return 0;
5717 }
5718
5719 static int
5720 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5721                                struct action *action,
5722                                char **tokens,
5723                                int n_tokens,
5724                                struct instruction *instr,
5725                                struct instruction_data *data)
5726 {
5727         struct action *a;
5728
5729         CHECK(!action, EINVAL);
5730         CHECK(n_tokens == 3, EINVAL);
5731
5732         strcpy(data->jmp_label, tokens[1]);
5733
5734         a = action_find(p, tokens[2]);
5735         CHECK(a, EINVAL);
5736
5737         instr->type = INSTR_JMP_ACTION_HIT;
5738         instr->jmp.ip = NULL; /* Resolved later. */
5739         instr->jmp.action_id = a->id;
5740         return 0;
5741 }
5742
5743 static int
5744 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5745                                 struct action *action,
5746                                 char **tokens,
5747                                 int n_tokens,
5748                                 struct instruction *instr,
5749                                 struct instruction_data *data)
5750 {
5751         struct action *a;
5752
5753         CHECK(!action, EINVAL);
5754         CHECK(n_tokens == 3, EINVAL);
5755
5756         strcpy(data->jmp_label, tokens[1]);
5757
5758         a = action_find(p, tokens[2]);
5759         CHECK(a, EINVAL);
5760
5761         instr->type = INSTR_JMP_ACTION_MISS;
5762         instr->jmp.ip = NULL; /* Resolved later. */
5763         instr->jmp.action_id = a->id;
5764         return 0;
5765 }
5766
5767 static int
5768 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5769                        struct action *action,
5770                        char **tokens,
5771                        int n_tokens,
5772                        struct instruction *instr,
5773                        struct instruction_data *data)
5774 {
5775         char *a = tokens[2], *b = tokens[3];
5776         struct field *fa, *fb;
5777         uint64_t b_val;
5778         uint32_t a_struct_id, b_struct_id;
5779
5780         CHECK(n_tokens == 4, EINVAL);
5781
5782         strcpy(data->jmp_label, tokens[1]);
5783
5784         fa = struct_field_parse(p, action, a, &a_struct_id);
5785         CHECK(fa, EINVAL);
5786         CHECK(!fa->var_size, EINVAL);
5787
5788         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5789         fb = struct_field_parse(p, action, b, &b_struct_id);
5790         if (fb) {
5791                 CHECK(!fb->var_size, EINVAL);
5792
5793                 instr->type = INSTR_JMP_EQ;
5794                 if (a[0] != 'h' && b[0] == 'h')
5795                         instr->type = INSTR_JMP_EQ_MH;
5796                 if (a[0] == 'h' && b[0] != 'h')
5797                         instr->type = INSTR_JMP_EQ_HM;
5798                 if (a[0] == 'h' && b[0] == 'h')
5799                         instr->type = INSTR_JMP_EQ_HH;
5800                 instr->jmp.ip = NULL; /* Resolved later. */
5801
5802                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5803                 instr->jmp.a.n_bits = fa->n_bits;
5804                 instr->jmp.a.offset = fa->offset / 8;
5805                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5806                 instr->jmp.b.n_bits = fb->n_bits;
5807                 instr->jmp.b.offset = fb->offset / 8;
5808                 return 0;
5809         }
5810
5811         /* JMP_EQ_I. */
5812         b_val = strtoull(b, &b, 0);
5813         CHECK(!b[0], EINVAL);
5814
5815         if (a[0] == 'h')
5816                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5817
5818         instr->type = INSTR_JMP_EQ_I;
5819         instr->jmp.ip = NULL; /* Resolved later. */
5820         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5821         instr->jmp.a.n_bits = fa->n_bits;
5822         instr->jmp.a.offset = fa->offset / 8;
5823         instr->jmp.b_val = b_val;
5824         return 0;
5825 }
5826
5827 static int
5828 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5829                         struct action *action,
5830                         char **tokens,
5831                         int n_tokens,
5832                         struct instruction *instr,
5833                         struct instruction_data *data)
5834 {
5835         char *a = tokens[2], *b = tokens[3];
5836         struct field *fa, *fb;
5837         uint64_t b_val;
5838         uint32_t a_struct_id, b_struct_id;
5839
5840         CHECK(n_tokens == 4, EINVAL);
5841
5842         strcpy(data->jmp_label, tokens[1]);
5843
5844         fa = struct_field_parse(p, action, a, &a_struct_id);
5845         CHECK(fa, EINVAL);
5846         CHECK(!fa->var_size, EINVAL);
5847
5848         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5849         fb = struct_field_parse(p, action, b, &b_struct_id);
5850         if (fb) {
5851                 CHECK(!fb->var_size, EINVAL);
5852
5853                 instr->type = INSTR_JMP_NEQ;
5854                 if (a[0] != 'h' && b[0] == 'h')
5855                         instr->type = INSTR_JMP_NEQ_MH;
5856                 if (a[0] == 'h' && b[0] != 'h')
5857                         instr->type = INSTR_JMP_NEQ_HM;
5858                 if (a[0] == 'h' && b[0] == 'h')
5859                         instr->type = INSTR_JMP_NEQ_HH;
5860                 instr->jmp.ip = NULL; /* Resolved later. */
5861
5862                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5863                 instr->jmp.a.n_bits = fa->n_bits;
5864                 instr->jmp.a.offset = fa->offset / 8;
5865                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5866                 instr->jmp.b.n_bits = fb->n_bits;
5867                 instr->jmp.b.offset = fb->offset / 8;
5868                 return 0;
5869         }
5870
5871         /* JMP_NEQ_I. */
5872         b_val = strtoull(b, &b, 0);
5873         CHECK(!b[0], EINVAL);
5874
5875         if (a[0] == 'h')
5876                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5877
5878         instr->type = INSTR_JMP_NEQ_I;
5879         instr->jmp.ip = NULL; /* Resolved later. */
5880         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5881         instr->jmp.a.n_bits = fa->n_bits;
5882         instr->jmp.a.offset = fa->offset / 8;
5883         instr->jmp.b_val = b_val;
5884         return 0;
5885 }
5886
5887 static int
5888 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5889                        struct action *action,
5890                        char **tokens,
5891                        int n_tokens,
5892                        struct instruction *instr,
5893                        struct instruction_data *data)
5894 {
5895         char *a = tokens[2], *b = tokens[3];
5896         struct field *fa, *fb;
5897         uint64_t b_val;
5898         uint32_t a_struct_id, b_struct_id;
5899
5900         CHECK(n_tokens == 4, EINVAL);
5901
5902         strcpy(data->jmp_label, tokens[1]);
5903
5904         fa = struct_field_parse(p, action, a, &a_struct_id);
5905         CHECK(fa, EINVAL);
5906         CHECK(!fa->var_size, EINVAL);
5907
5908         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5909         fb = struct_field_parse(p, action, b, &b_struct_id);
5910         if (fb) {
5911                 CHECK(!fb->var_size, EINVAL);
5912
5913                 instr->type = INSTR_JMP_LT;
5914                 if (a[0] == 'h' && b[0] != 'h')
5915                         instr->type = INSTR_JMP_LT_HM;
5916                 if (a[0] != 'h' && b[0] == 'h')
5917                         instr->type = INSTR_JMP_LT_MH;
5918                 if (a[0] == 'h' && b[0] == 'h')
5919                         instr->type = INSTR_JMP_LT_HH;
5920                 instr->jmp.ip = NULL; /* Resolved later. */
5921
5922                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5923                 instr->jmp.a.n_bits = fa->n_bits;
5924                 instr->jmp.a.offset = fa->offset / 8;
5925                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5926                 instr->jmp.b.n_bits = fb->n_bits;
5927                 instr->jmp.b.offset = fb->offset / 8;
5928                 return 0;
5929         }
5930
5931         /* JMP_LT_MI, JMP_LT_HI. */
5932         b_val = strtoull(b, &b, 0);
5933         CHECK(!b[0], EINVAL);
5934
5935         instr->type = INSTR_JMP_LT_MI;
5936         if (a[0] == 'h')
5937                 instr->type = INSTR_JMP_LT_HI;
5938         instr->jmp.ip = NULL; /* Resolved later. */
5939
5940         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5941         instr->jmp.a.n_bits = fa->n_bits;
5942         instr->jmp.a.offset = fa->offset / 8;
5943         instr->jmp.b_val = b_val;
5944         return 0;
5945 }
5946
5947 static int
5948 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5949                        struct action *action,
5950                        char **tokens,
5951                        int n_tokens,
5952                        struct instruction *instr,
5953                        struct instruction_data *data)
5954 {
5955         char *a = tokens[2], *b = tokens[3];
5956         struct field *fa, *fb;
5957         uint64_t b_val;
5958         uint32_t a_struct_id, b_struct_id;
5959
5960         CHECK(n_tokens == 4, EINVAL);
5961
5962         strcpy(data->jmp_label, tokens[1]);
5963
5964         fa = struct_field_parse(p, action, a, &a_struct_id);
5965         CHECK(fa, EINVAL);
5966         CHECK(!fa->var_size, EINVAL);
5967
5968         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5969         fb = struct_field_parse(p, action, b, &b_struct_id);
5970         if (fb) {
5971                 CHECK(!fb->var_size, EINVAL);
5972
5973                 instr->type = INSTR_JMP_GT;
5974                 if (a[0] == 'h' && b[0] != 'h')
5975                         instr->type = INSTR_JMP_GT_HM;
5976                 if (a[0] != 'h' && b[0] == 'h')
5977                         instr->type = INSTR_JMP_GT_MH;
5978                 if (a[0] == 'h' && b[0] == 'h')
5979                         instr->type = INSTR_JMP_GT_HH;
5980                 instr->jmp.ip = NULL; /* Resolved later. */
5981
5982                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5983                 instr->jmp.a.n_bits = fa->n_bits;
5984                 instr->jmp.a.offset = fa->offset / 8;
5985                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5986                 instr->jmp.b.n_bits = fb->n_bits;
5987                 instr->jmp.b.offset = fb->offset / 8;
5988                 return 0;
5989         }
5990
5991         /* JMP_GT_MI, JMP_GT_HI. */
5992         b_val = strtoull(b, &b, 0);
5993         CHECK(!b[0], EINVAL);
5994
5995         instr->type = INSTR_JMP_GT_MI;
5996         if (a[0] == 'h')
5997                 instr->type = INSTR_JMP_GT_HI;
5998         instr->jmp.ip = NULL; /* Resolved later. */
5999
6000         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6001         instr->jmp.a.n_bits = fa->n_bits;
6002         instr->jmp.a.offset = fa->offset / 8;
6003         instr->jmp.b_val = b_val;
6004         return 0;
6005 }
6006
6007 static inline void
6008 instr_jmp_exec(struct rte_swx_pipeline *p)
6009 {
6010         struct thread *t = &p->threads[p->thread_id];
6011         struct instruction *ip = t->ip;
6012
6013         TRACE("[Thread %2u] jmp\n", p->thread_id);
6014
6015         thread_ip_set(t, ip->jmp.ip);
6016 }
6017
6018 static inline void
6019 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6020 {
6021         struct thread *t = &p->threads[p->thread_id];
6022         struct instruction *ip = t->ip;
6023         uint32_t header_id = ip->jmp.header_id;
6024
6025         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6026
6027         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6028 }
6029
6030 static inline void
6031 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6032 {
6033         struct thread *t = &p->threads[p->thread_id];
6034         struct instruction *ip = t->ip;
6035         uint32_t header_id = ip->jmp.header_id;
6036
6037         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6038
6039         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6040 }
6041
6042 static inline void
6043 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6044 {
6045         struct thread *t = &p->threads[p->thread_id];
6046         struct instruction *ip = t->ip;
6047         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6048
6049         TRACE("[Thread %2u] jmph\n", p->thread_id);
6050
6051         t->ip = ip_next[t->hit];
6052 }
6053
6054 static inline void
6055 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6056 {
6057         struct thread *t = &p->threads[p->thread_id];
6058         struct instruction *ip = t->ip;
6059         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6060
6061         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6062
6063         t->ip = ip_next[t->hit];
6064 }
6065
6066 static inline void
6067 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6068 {
6069         struct thread *t = &p->threads[p->thread_id];
6070         struct instruction *ip = t->ip;
6071
6072         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6073
6074         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6075 }
6076
6077 static inline void
6078 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6079 {
6080         struct thread *t = &p->threads[p->thread_id];
6081         struct instruction *ip = t->ip;
6082
6083         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6084
6085         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6086 }
6087
6088 static inline void
6089 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6090 {
6091         struct thread *t = &p->threads[p->thread_id];
6092         struct instruction *ip = t->ip;
6093
6094         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6095
6096         JMP_CMP(t, ip, ==);
6097 }
6098
6099 static inline void
6100 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6101 {
6102         struct thread *t = &p->threads[p->thread_id];
6103         struct instruction *ip = t->ip;
6104
6105         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6106
6107         JMP_CMP_MH(t, ip, ==);
6108 }
6109
6110 static inline void
6111 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6112 {
6113         struct thread *t = &p->threads[p->thread_id];
6114         struct instruction *ip = t->ip;
6115
6116         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6117
6118         JMP_CMP_HM(t, ip, ==);
6119 }
6120
6121 static inline void
6122 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6123 {
6124         struct thread *t = &p->threads[p->thread_id];
6125         struct instruction *ip = t->ip;
6126
6127         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6128
6129         JMP_CMP_HH_FAST(t, ip, ==);
6130 }
6131
6132 static inline void
6133 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6134 {
6135         struct thread *t = &p->threads[p->thread_id];
6136         struct instruction *ip = t->ip;
6137
6138         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6139
6140         JMP_CMP_I(t, ip, ==);
6141 }
6142
6143 static inline void
6144 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6145 {
6146         struct thread *t = &p->threads[p->thread_id];
6147         struct instruction *ip = t->ip;
6148
6149         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6150
6151         JMP_CMP(t, ip, !=);
6152 }
6153
6154 static inline void
6155 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6156 {
6157         struct thread *t = &p->threads[p->thread_id];
6158         struct instruction *ip = t->ip;
6159
6160         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6161
6162         JMP_CMP_MH(t, ip, !=);
6163 }
6164
6165 static inline void
6166 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6167 {
6168         struct thread *t = &p->threads[p->thread_id];
6169         struct instruction *ip = t->ip;
6170
6171         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6172
6173         JMP_CMP_HM(t, ip, !=);
6174 }
6175
6176 static inline void
6177 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6178 {
6179         struct thread *t = &p->threads[p->thread_id];
6180         struct instruction *ip = t->ip;
6181
6182         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6183
6184         JMP_CMP_HH_FAST(t, ip, !=);
6185 }
6186
6187 static inline void
6188 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6189 {
6190         struct thread *t = &p->threads[p->thread_id];
6191         struct instruction *ip = t->ip;
6192
6193         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6194
6195         JMP_CMP_I(t, ip, !=);
6196 }
6197
6198 static inline void
6199 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6200 {
6201         struct thread *t = &p->threads[p->thread_id];
6202         struct instruction *ip = t->ip;
6203
6204         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6205
6206         JMP_CMP(t, ip, <);
6207 }
6208
6209 static inline void
6210 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6211 {
6212         struct thread *t = &p->threads[p->thread_id];
6213         struct instruction *ip = t->ip;
6214
6215         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6216
6217         JMP_CMP_MH(t, ip, <);
6218 }
6219
6220 static inline void
6221 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6222 {
6223         struct thread *t = &p->threads[p->thread_id];
6224         struct instruction *ip = t->ip;
6225
6226         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6227
6228         JMP_CMP_HM(t, ip, <);
6229 }
6230
6231 static inline void
6232 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6233 {
6234         struct thread *t = &p->threads[p->thread_id];
6235         struct instruction *ip = t->ip;
6236
6237         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6238
6239         JMP_CMP_HH(t, ip, <);
6240 }
6241
6242 static inline void
6243 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6244 {
6245         struct thread *t = &p->threads[p->thread_id];
6246         struct instruction *ip = t->ip;
6247
6248         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6249
6250         JMP_CMP_MI(t, ip, <);
6251 }
6252
6253 static inline void
6254 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6255 {
6256         struct thread *t = &p->threads[p->thread_id];
6257         struct instruction *ip = t->ip;
6258
6259         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6260
6261         JMP_CMP_HI(t, ip, <);
6262 }
6263
6264 static inline void
6265 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6266 {
6267         struct thread *t = &p->threads[p->thread_id];
6268         struct instruction *ip = t->ip;
6269
6270         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6271
6272         JMP_CMP(t, ip, >);
6273 }
6274
6275 static inline void
6276 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6277 {
6278         struct thread *t = &p->threads[p->thread_id];
6279         struct instruction *ip = t->ip;
6280
6281         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6282
6283         JMP_CMP_MH(t, ip, >);
6284 }
6285
6286 static inline void
6287 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6288 {
6289         struct thread *t = &p->threads[p->thread_id];
6290         struct instruction *ip = t->ip;
6291
6292         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6293
6294         JMP_CMP_HM(t, ip, >);
6295 }
6296
6297 static inline void
6298 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6299 {
6300         struct thread *t = &p->threads[p->thread_id];
6301         struct instruction *ip = t->ip;
6302
6303         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6304
6305         JMP_CMP_HH(t, ip, >);
6306 }
6307
6308 static inline void
6309 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6310 {
6311         struct thread *t = &p->threads[p->thread_id];
6312         struct instruction *ip = t->ip;
6313
6314         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6315
6316         JMP_CMP_MI(t, ip, >);
6317 }
6318
6319 static inline void
6320 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6321 {
6322         struct thread *t = &p->threads[p->thread_id];
6323         struct instruction *ip = t->ip;
6324
6325         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6326
6327         JMP_CMP_HI(t, ip, >);
6328 }
6329
6330 /*
6331  * return.
6332  */
6333 static int
6334 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6335                        struct action *action,
6336                        char **tokens __rte_unused,
6337                        int n_tokens,
6338                        struct instruction *instr,
6339                        struct instruction_data *data __rte_unused)
6340 {
6341         CHECK(action, EINVAL);
6342         CHECK(n_tokens == 1, EINVAL);
6343
6344         instr->type = INSTR_RETURN;
6345         return 0;
6346 }
6347
6348 static inline void
6349 instr_return_exec(struct rte_swx_pipeline *p)
6350 {
6351         struct thread *t = &p->threads[p->thread_id];
6352
6353         TRACE("[Thread %2u] return\n", p->thread_id);
6354
6355         t->ip = t->ret;
6356 }
6357
6358 static int
6359 instr_translate(struct rte_swx_pipeline *p,
6360                 struct action *action,
6361                 char *string,
6362                 struct instruction *instr,
6363                 struct instruction_data *data)
6364 {
6365         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6366         int n_tokens = 0, tpos = 0;
6367
6368         /* Parse the instruction string into tokens. */
6369         for ( ; ; ) {
6370                 char *token;
6371
6372                 token = strtok_r(string, " \t\v", &string);
6373                 if (!token)
6374                         break;
6375
6376                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6377                 CHECK_NAME(token, EINVAL);
6378
6379                 tokens[n_tokens] = token;
6380                 n_tokens++;
6381         }
6382
6383         CHECK(n_tokens, EINVAL);
6384
6385         /* Handle the optional instruction label. */
6386         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6387                 strcpy(data->label, tokens[0]);
6388
6389                 tpos += 2;
6390                 CHECK(n_tokens - tpos, EINVAL);
6391         }
6392
6393         /* Identify the instruction type. */
6394         if (!strcmp(tokens[tpos], "rx"))
6395                 return instr_rx_translate(p,
6396                                           action,
6397                                           &tokens[tpos],
6398                                           n_tokens - tpos,
6399                                           instr,
6400                                           data);
6401
6402         if (!strcmp(tokens[tpos], "tx"))
6403                 return instr_tx_translate(p,
6404                                           action,
6405                                           &tokens[tpos],
6406                                           n_tokens - tpos,
6407                                           instr,
6408                                           data);
6409
6410         if (!strcmp(tokens[tpos], "drop"))
6411                 return instr_drop_translate(p,
6412                                             action,
6413                                             &tokens[tpos],
6414                                             n_tokens - tpos,
6415                                             instr,
6416                                             data);
6417
6418         if (!strcmp(tokens[tpos], "extract"))
6419                 return instr_hdr_extract_translate(p,
6420                                                    action,
6421                                                    &tokens[tpos],
6422                                                    n_tokens - tpos,
6423                                                    instr,
6424                                                    data);
6425
6426         if (!strcmp(tokens[tpos], "lookahead"))
6427                 return instr_hdr_lookahead_translate(p,
6428                                                      action,
6429                                                      &tokens[tpos],
6430                                                      n_tokens - tpos,
6431                                                      instr,
6432                                                      data);
6433
6434         if (!strcmp(tokens[tpos], "emit"))
6435                 return instr_hdr_emit_translate(p,
6436                                                 action,
6437                                                 &tokens[tpos],
6438                                                 n_tokens - tpos,
6439                                                 instr,
6440                                                 data);
6441
6442         if (!strcmp(tokens[tpos], "validate"))
6443                 return instr_hdr_validate_translate(p,
6444                                                     action,
6445                                                     &tokens[tpos],
6446                                                     n_tokens - tpos,
6447                                                     instr,
6448                                                     data);
6449
6450         if (!strcmp(tokens[tpos], "invalidate"))
6451                 return instr_hdr_invalidate_translate(p,
6452                                                       action,
6453                                                       &tokens[tpos],
6454                                                       n_tokens - tpos,
6455                                                       instr,
6456                                                       data);
6457
6458         if (!strcmp(tokens[tpos], "mov"))
6459                 return instr_mov_translate(p,
6460                                            action,
6461                                            &tokens[tpos],
6462                                            n_tokens - tpos,
6463                                            instr,
6464                                            data);
6465
6466         if (!strcmp(tokens[tpos], "add"))
6467                 return instr_alu_add_translate(p,
6468                                                action,
6469                                                &tokens[tpos],
6470                                                n_tokens - tpos,
6471                                                instr,
6472                                                data);
6473
6474         if (!strcmp(tokens[tpos], "sub"))
6475                 return instr_alu_sub_translate(p,
6476                                                action,
6477                                                &tokens[tpos],
6478                                                n_tokens - tpos,
6479                                                instr,
6480                                                data);
6481
6482         if (!strcmp(tokens[tpos], "ckadd"))
6483                 return instr_alu_ckadd_translate(p,
6484                                                  action,
6485                                                  &tokens[tpos],
6486                                                  n_tokens - tpos,
6487                                                  instr,
6488                                                  data);
6489
6490         if (!strcmp(tokens[tpos], "cksub"))
6491                 return instr_alu_cksub_translate(p,
6492                                                  action,
6493                                                  &tokens[tpos],
6494                                                  n_tokens - tpos,
6495                                                  instr,
6496                                                  data);
6497
6498         if (!strcmp(tokens[tpos], "and"))
6499                 return instr_alu_and_translate(p,
6500                                                action,
6501                                                &tokens[tpos],
6502                                                n_tokens - tpos,
6503                                                instr,
6504                                                data);
6505
6506         if (!strcmp(tokens[tpos], "or"))
6507                 return instr_alu_or_translate(p,
6508                                               action,
6509                                               &tokens[tpos],
6510                                               n_tokens - tpos,
6511                                               instr,
6512                                               data);
6513
6514         if (!strcmp(tokens[tpos], "xor"))
6515                 return instr_alu_xor_translate(p,
6516                                                action,
6517                                                &tokens[tpos],
6518                                                n_tokens - tpos,
6519                                                instr,
6520                                                data);
6521
6522         if (!strcmp(tokens[tpos], "shl"))
6523                 return instr_alu_shl_translate(p,
6524                                                action,
6525                                                &tokens[tpos],
6526                                                n_tokens - tpos,
6527                                                instr,
6528                                                data);
6529
6530         if (!strcmp(tokens[tpos], "shr"))
6531                 return instr_alu_shr_translate(p,
6532                                                action,
6533                                                &tokens[tpos],
6534                                                n_tokens - tpos,
6535                                                instr,
6536                                                data);
6537
6538         if (!strcmp(tokens[tpos], "regprefetch"))
6539                 return instr_regprefetch_translate(p,
6540                                                    action,
6541                                                    &tokens[tpos],
6542                                                    n_tokens - tpos,
6543                                                    instr,
6544                                                    data);
6545
6546         if (!strcmp(tokens[tpos], "regrd"))
6547                 return instr_regrd_translate(p,
6548                                              action,
6549                                              &tokens[tpos],
6550                                              n_tokens - tpos,
6551                                              instr,
6552                                              data);
6553
6554         if (!strcmp(tokens[tpos], "regwr"))
6555                 return instr_regwr_translate(p,
6556                                              action,
6557                                              &tokens[tpos],
6558                                              n_tokens - tpos,
6559                                              instr,
6560                                              data);
6561
6562         if (!strcmp(tokens[tpos], "regadd"))
6563                 return instr_regadd_translate(p,
6564                                               action,
6565                                               &tokens[tpos],
6566                                               n_tokens - tpos,
6567                                               instr,
6568                                               data);
6569
6570         if (!strcmp(tokens[tpos], "metprefetch"))
6571                 return instr_metprefetch_translate(p,
6572                                                    action,
6573                                                    &tokens[tpos],
6574                                                    n_tokens - tpos,
6575                                                    instr,
6576                                                    data);
6577
6578         if (!strcmp(tokens[tpos], "meter"))
6579                 return instr_meter_translate(p,
6580                                              action,
6581                                              &tokens[tpos],
6582                                              n_tokens - tpos,
6583                                              instr,
6584                                              data);
6585
6586         if (!strcmp(tokens[tpos], "table"))
6587                 return instr_table_translate(p,
6588                                              action,
6589                                              &tokens[tpos],
6590                                              n_tokens - tpos,
6591                                              instr,
6592                                              data);
6593
6594         if (!strcmp(tokens[tpos], "learn"))
6595                 return instr_learn_translate(p,
6596                                              action,
6597                                              &tokens[tpos],
6598                                              n_tokens - tpos,
6599                                              instr,
6600                                              data);
6601
6602         if (!strcmp(tokens[tpos], "forget"))
6603                 return instr_forget_translate(p,
6604                                               action,
6605                                               &tokens[tpos],
6606                                               n_tokens - tpos,
6607                                               instr,
6608                                               data);
6609
6610         if (!strcmp(tokens[tpos], "extern"))
6611                 return instr_extern_translate(p,
6612                                               action,
6613                                               &tokens[tpos],
6614                                               n_tokens - tpos,
6615                                               instr,
6616                                               data);
6617
6618         if (!strcmp(tokens[tpos], "jmp"))
6619                 return instr_jmp_translate(p,
6620                                            action,
6621                                            &tokens[tpos],
6622                                            n_tokens - tpos,
6623                                            instr,
6624                                            data);
6625
6626         if (!strcmp(tokens[tpos], "jmpv"))
6627                 return instr_jmp_valid_translate(p,
6628                                                  action,
6629                                                  &tokens[tpos],
6630                                                  n_tokens - tpos,
6631                                                  instr,
6632                                                  data);
6633
6634         if (!strcmp(tokens[tpos], "jmpnv"))
6635                 return instr_jmp_invalid_translate(p,
6636                                                    action,
6637                                                    &tokens[tpos],
6638                                                    n_tokens - tpos,
6639                                                    instr,
6640                                                    data);
6641
6642         if (!strcmp(tokens[tpos], "jmph"))
6643                 return instr_jmp_hit_translate(p,
6644                                                action,
6645                                                &tokens[tpos],
6646                                                n_tokens - tpos,
6647                                                instr,
6648                                                data);
6649
6650         if (!strcmp(tokens[tpos], "jmpnh"))
6651                 return instr_jmp_miss_translate(p,
6652                                                 action,
6653                                                 &tokens[tpos],
6654                                                 n_tokens - tpos,
6655                                                 instr,
6656                                                 data);
6657
6658         if (!strcmp(tokens[tpos], "jmpa"))
6659                 return instr_jmp_action_hit_translate(p,
6660                                                       action,
6661                                                       &tokens[tpos],
6662                                                       n_tokens - tpos,
6663                                                       instr,
6664                                                       data);
6665
6666         if (!strcmp(tokens[tpos], "jmpna"))
6667                 return instr_jmp_action_miss_translate(p,
6668                                                        action,
6669                                                        &tokens[tpos],
6670                                                        n_tokens - tpos,
6671                                                        instr,
6672                                                        data);
6673
6674         if (!strcmp(tokens[tpos], "jmpeq"))
6675                 return instr_jmp_eq_translate(p,
6676                                               action,
6677                                               &tokens[tpos],
6678                                               n_tokens - tpos,
6679                                               instr,
6680                                               data);
6681
6682         if (!strcmp(tokens[tpos], "jmpneq"))
6683                 return instr_jmp_neq_translate(p,
6684                                                action,
6685                                                &tokens[tpos],
6686                                                n_tokens - tpos,
6687                                                instr,
6688                                                data);
6689
6690         if (!strcmp(tokens[tpos], "jmplt"))
6691                 return instr_jmp_lt_translate(p,
6692                                               action,
6693                                               &tokens[tpos],
6694                                               n_tokens - tpos,
6695                                               instr,
6696                                               data);
6697
6698         if (!strcmp(tokens[tpos], "jmpgt"))
6699                 return instr_jmp_gt_translate(p,
6700                                               action,
6701                                               &tokens[tpos],
6702                                               n_tokens - tpos,
6703                                               instr,
6704                                               data);
6705
6706         if (!strcmp(tokens[tpos], "return"))
6707                 return instr_return_translate(p,
6708                                               action,
6709                                               &tokens[tpos],
6710                                               n_tokens - tpos,
6711                                               instr,
6712                                               data);
6713
6714         CHECK(0, EINVAL);
6715 }
6716
6717 static struct instruction_data *
6718 label_find(struct instruction_data *data, uint32_t n, const char *label)
6719 {
6720         uint32_t i;
6721
6722         for (i = 0; i < n; i++)
6723                 if (!strcmp(label, data[i].label))
6724                         return &data[i];
6725
6726         return NULL;
6727 }
6728
6729 static uint32_t
6730 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6731 {
6732         uint32_t count = 0, i;
6733
6734         if (!label[0])
6735                 return 0;
6736
6737         for (i = 0; i < n; i++)
6738                 if (!strcmp(label, data[i].jmp_label))
6739                         count++;
6740
6741         return count;
6742 }
6743
6744 static int
6745 instr_label_check(struct instruction_data *instruction_data,
6746                   uint32_t n_instructions)
6747 {
6748         uint32_t i;
6749
6750         /* Check that all instruction labels are unique. */
6751         for (i = 0; i < n_instructions; i++) {
6752                 struct instruction_data *data = &instruction_data[i];
6753                 char *label = data->label;
6754                 uint32_t j;
6755
6756                 if (!label[0])
6757                         continue;
6758
6759                 for (j = i + 1; j < n_instructions; j++)
6760                         CHECK(strcmp(label, data[j].label), EINVAL);
6761         }
6762
6763         /* Get users for each instruction label. */
6764         for (i = 0; i < n_instructions; i++) {
6765                 struct instruction_data *data = &instruction_data[i];
6766                 char *label = data->label;
6767
6768                 data->n_users = label_is_used(instruction_data,
6769                                               n_instructions,
6770                                               label);
6771         }
6772
6773         return 0;
6774 }
6775
6776 static int
6777 instr_jmp_resolve(struct instruction *instructions,
6778                   struct instruction_data *instruction_data,
6779                   uint32_t n_instructions)
6780 {
6781         uint32_t i;
6782
6783         for (i = 0; i < n_instructions; i++) {
6784                 struct instruction *instr = &instructions[i];
6785                 struct instruction_data *data = &instruction_data[i];
6786                 struct instruction_data *found;
6787
6788                 if (!instruction_is_jmp(instr))
6789                         continue;
6790
6791                 found = label_find(instruction_data,
6792                                    n_instructions,
6793                                    data->jmp_label);
6794                 CHECK(found, EINVAL);
6795
6796                 instr->jmp.ip = &instructions[found - instruction_data];
6797         }
6798
6799         return 0;
6800 }
6801
6802 static int
6803 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6804              struct action *a,
6805              struct instruction *instr,
6806              struct instruction_data *data __rte_unused,
6807              uint32_t n_instructions)
6808 {
6809         if (!a) {
6810                 enum instruction_type type;
6811                 uint32_t i;
6812
6813                 /* Check that the first instruction is rx. */
6814                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6815
6816                 /* Check that there is at least one tx instruction. */
6817                 for (i = 0; i < n_instructions; i++) {
6818                         type = instr[i].type;
6819
6820                         if (instruction_is_tx(type))
6821                                 break;
6822                 }
6823                 CHECK(i < n_instructions, EINVAL);
6824
6825                 /* Check that the last instruction is either tx or unconditional
6826                  * jump.
6827                  */
6828                 type = instr[n_instructions - 1].type;
6829                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6830         }
6831
6832         if (a) {
6833                 enum instruction_type type;
6834                 uint32_t i;
6835
6836                 /* Check that there is at least one return or tx instruction. */
6837                 for (i = 0; i < n_instructions; i++) {
6838                         type = instr[i].type;
6839
6840                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6841                                 break;
6842                 }
6843                 CHECK(i < n_instructions, EINVAL);
6844         }
6845
6846         return 0;
6847 }
6848
6849 static uint32_t
6850 instr_compact(struct instruction *instructions,
6851               struct instruction_data *instruction_data,
6852               uint32_t n_instructions)
6853 {
6854         uint32_t i, pos = 0;
6855
6856         /* Eliminate the invalid instructions that have been optimized out. */
6857         for (i = 0; i < n_instructions; i++) {
6858                 struct instruction *instr = &instructions[i];
6859                 struct instruction_data *data = &instruction_data[i];
6860
6861                 if (data->invalid)
6862                         continue;
6863
6864                 if (i != pos) {
6865                         memcpy(&instructions[pos], instr, sizeof(*instr));
6866                         memcpy(&instruction_data[pos], data, sizeof(*data));
6867                 }
6868
6869                 pos++;
6870         }
6871
6872         return pos;
6873 }
6874
6875 static int
6876 instr_pattern_extract_many_search(struct instruction *instr,
6877                                   struct instruction_data *data,
6878                                   uint32_t n_instr,
6879                                   uint32_t *n_pattern_instr)
6880 {
6881         uint32_t i;
6882
6883         for (i = 0; i < n_instr; i++) {
6884                 if (data[i].invalid)
6885                         break;
6886
6887                 if (instr[i].type != INSTR_HDR_EXTRACT)
6888                         break;
6889
6890                 if (i == RTE_DIM(instr->io.hdr.header_id))
6891                         break;
6892
6893                 if (i && data[i].n_users)
6894                         break;
6895         }
6896
6897         if (i < 2)
6898                 return 0;
6899
6900         *n_pattern_instr = i;
6901         return 1;
6902 }
6903
6904 static void
6905 instr_pattern_extract_many_replace(struct instruction *instr,
6906                                    struct instruction_data *data,
6907                                    uint32_t n_instr)
6908 {
6909         uint32_t i;
6910
6911         for (i = 1; i < n_instr; i++) {
6912                 instr[0].type++;
6913                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6914                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6915                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6916
6917                 data[i].invalid = 1;
6918         }
6919 }
6920
6921 static uint32_t
6922 instr_pattern_extract_many_optimize(struct instruction *instructions,
6923                                     struct instruction_data *instruction_data,
6924                                     uint32_t n_instructions)
6925 {
6926         uint32_t i;
6927
6928         for (i = 0; i < n_instructions; ) {
6929                 struct instruction *instr = &instructions[i];
6930                 struct instruction_data *data = &instruction_data[i];
6931                 uint32_t n_instr = 0;
6932                 int detected;
6933
6934                 /* Extract many. */
6935                 detected = instr_pattern_extract_many_search(instr,
6936                                                              data,
6937                                                              n_instructions - i,
6938                                                              &n_instr);
6939                 if (detected) {
6940                         instr_pattern_extract_many_replace(instr,
6941                                                            data,
6942                                                            n_instr);
6943                         i += n_instr;
6944                         continue;
6945                 }
6946
6947                 /* No pattern starting at the current instruction. */
6948                 i++;
6949         }
6950
6951         /* Eliminate the invalid instructions that have been optimized out. */
6952         n_instructions = instr_compact(instructions,
6953                                        instruction_data,
6954                                        n_instructions);
6955
6956         return n_instructions;
6957 }
6958
6959 static int
6960 instr_pattern_emit_many_tx_search(struct instruction *instr,
6961                                   struct instruction_data *data,
6962                                   uint32_t n_instr,
6963                                   uint32_t *n_pattern_instr)
6964 {
6965         uint32_t i;
6966
6967         for (i = 0; i < n_instr; i++) {
6968                 if (data[i].invalid)
6969                         break;
6970
6971                 if (instr[i].type != INSTR_HDR_EMIT)
6972                         break;
6973
6974                 if (i == RTE_DIM(instr->io.hdr.header_id))
6975                         break;
6976
6977                 if (i && data[i].n_users)
6978                         break;
6979         }
6980
6981         if (!i)
6982                 return 0;
6983
6984         if (!instruction_is_tx(instr[i].type))
6985                 return 0;
6986
6987         if (data[i].n_users)
6988                 return 0;
6989
6990         i++;
6991
6992         *n_pattern_instr = i;
6993         return 1;
6994 }
6995
6996 static void
6997 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6998                                    struct instruction_data *data,
6999                                    uint32_t n_instr)
7000 {
7001         uint32_t i;
7002
7003         /* Any emit instruction in addition to the first one. */
7004         for (i = 1; i < n_instr - 1; i++) {
7005                 instr[0].type++;
7006                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7007                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7008                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7009
7010                 data[i].invalid = 1;
7011         }
7012
7013         /* The TX instruction is the last one in the pattern. */
7014         instr[0].type++;
7015         instr[0].io.io.offset = instr[i].io.io.offset;
7016         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7017         data[i].invalid = 1;
7018 }
7019
7020 static uint32_t
7021 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7022                                     struct instruction_data *instruction_data,
7023                                     uint32_t n_instructions)
7024 {
7025         uint32_t i;
7026
7027         for (i = 0; i < n_instructions; ) {
7028                 struct instruction *instr = &instructions[i];
7029                 struct instruction_data *data = &instruction_data[i];
7030                 uint32_t n_instr = 0;
7031                 int detected;
7032
7033                 /* Emit many + TX. */
7034                 detected = instr_pattern_emit_many_tx_search(instr,
7035                                                              data,
7036                                                              n_instructions - i,
7037                                                              &n_instr);
7038                 if (detected) {
7039                         instr_pattern_emit_many_tx_replace(instr,
7040                                                            data,
7041                                                            n_instr);
7042                         i += n_instr;
7043                         continue;
7044                 }
7045
7046                 /* No pattern starting at the current instruction. */
7047                 i++;
7048         }
7049
7050         /* Eliminate the invalid instructions that have been optimized out. */
7051         n_instructions = instr_compact(instructions,
7052                                        instruction_data,
7053                                        n_instructions);
7054
7055         return n_instructions;
7056 }
7057
7058 static uint32_t
7059 action_arg_src_mov_count(struct action *a,
7060                          uint32_t arg_id,
7061                          struct instruction *instructions,
7062                          struct instruction_data *instruction_data,
7063                          uint32_t n_instructions);
7064
7065 static int
7066 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7067                                       struct action *a,
7068                                       struct instruction *instr,
7069                                       struct instruction_data *data,
7070                                       uint32_t n_instr,
7071                                       struct instruction *instructions,
7072                                       struct instruction_data *instruction_data,
7073                                       uint32_t n_instructions,
7074                                       uint32_t *n_pattern_instr)
7075 {
7076         struct header *h;
7077         uint32_t src_field_id, i, j;
7078
7079         /* Prerequisites. */
7080         if (!a || !a->st)
7081                 return 0;
7082
7083         /* First instruction: MOV_HM. */
7084         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7085                 return 0;
7086
7087         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7088         if (!h || h->st->var_size)
7089                 return 0;
7090
7091         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7092                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7093                         break;
7094
7095         if (src_field_id == a->st->n_fields)
7096                 return 0;
7097
7098         if (instr[0].mov.dst.offset ||
7099             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7100             instr[0].mov.src.struct_id ||
7101             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7102             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7103                 return 0;
7104
7105         if ((n_instr < h->st->n_fields + 1) ||
7106              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7107                 return 0;
7108
7109         /* Subsequent instructions: MOV_HM. */
7110         for (i = 1; i < h->st->n_fields; i++)
7111                 if (data[i].invalid ||
7112                     data[i].n_users ||
7113                     (instr[i].type != INSTR_MOV_HM) ||
7114                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7115                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7116                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7117                     instr[i].mov.src.struct_id ||
7118                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7119                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7120                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7121                         return 0;
7122
7123         /* Last instruction: HDR_VALIDATE. */
7124         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7125             (instr[i].valid.header_id != h->id))
7126                 return 0;
7127
7128         /* Check that none of the action args that are used as source for this
7129          * DMA transfer are not used as source in any other mov instruction.
7130          */
7131         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7132                 uint32_t n_users;
7133
7134                 n_users = action_arg_src_mov_count(a,
7135                                                    j,
7136                                                    instructions,
7137                                                    instruction_data,
7138                                                    n_instructions);
7139                 if (n_users > 1)
7140                         return 0;
7141         }
7142
7143         *n_pattern_instr = 1 + i;
7144         return 1;
7145 }
7146
7147 static void
7148 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7149                                        struct action *a,
7150                                        struct instruction *instr,
7151                                        struct instruction_data *data,
7152                                        uint32_t n_instr)
7153 {
7154         struct header *h;
7155         uint32_t src_field_id, src_offset, i;
7156
7157         /* Read from the instructions before they are modified. */
7158         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7159         if (!h)
7160                 return;
7161
7162         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7163                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7164                         break;
7165
7166         if (src_field_id == a->st->n_fields)
7167                 return;
7168
7169         src_offset = instr[0].mov.src.offset;
7170
7171         /* Modify the instructions. */
7172         instr[0].type = INSTR_DMA_HT;
7173         instr[0].dma.dst.header_id[0] = h->id;
7174         instr[0].dma.dst.struct_id[0] = h->struct_id;
7175         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7176         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7177
7178         for (i = 1; i < n_instr; i++)
7179                 data[i].invalid = 1;
7180
7181         /* Update the endianness of the action arguments to header endianness. */
7182         for (i = 0; i < h->st->n_fields; i++)
7183                 a->args_endianness[src_field_id + i] = 1;
7184 }
7185
7186 static uint32_t
7187 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7188                                         struct action *a,
7189                                         struct instruction *instructions,
7190                                         struct instruction_data *instruction_data,
7191                                         uint32_t n_instructions)
7192 {
7193         uint32_t i;
7194
7195         if (!a || !a->st)
7196                 return n_instructions;
7197
7198         for (i = 0; i < n_instructions; ) {
7199                 struct instruction *instr = &instructions[i];
7200                 struct instruction_data *data = &instruction_data[i];
7201                 uint32_t n_instr = 0;
7202                 int detected;
7203
7204                 /* Mov all + validate. */
7205                 detected = instr_pattern_mov_all_validate_search(p,
7206                                                                  a,
7207                                                                  instr,
7208                                                                  data,
7209                                                                  n_instructions - i,
7210                                                                  instructions,
7211                                                                  instruction_data,
7212                                                                  n_instructions,
7213                                                                  &n_instr);
7214                 if (detected) {
7215                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7216                         i += n_instr;
7217                         continue;
7218                 }
7219
7220                 /* No pattern starting at the current instruction. */
7221                 i++;
7222         }
7223
7224         /* Eliminate the invalid instructions that have been optimized out. */
7225         n_instructions = instr_compact(instructions,
7226                                        instruction_data,
7227                                        n_instructions);
7228
7229         return n_instructions;
7230 }
7231
7232 static int
7233 instr_pattern_dma_many_search(struct instruction *instr,
7234                               struct instruction_data *data,
7235                               uint32_t n_instr,
7236                               uint32_t *n_pattern_instr)
7237 {
7238         uint32_t i;
7239
7240         for (i = 0; i < n_instr; i++) {
7241                 if (data[i].invalid)
7242                         break;
7243
7244                 if (instr[i].type != INSTR_DMA_HT)
7245                         break;
7246
7247                 if (i == RTE_DIM(instr->dma.dst.header_id))
7248                         break;
7249
7250                 if (i && data[i].n_users)
7251                         break;
7252         }
7253
7254         if (i < 2)
7255                 return 0;
7256
7257         *n_pattern_instr = i;
7258         return 1;
7259 }
7260
7261 static void
7262 instr_pattern_dma_many_replace(struct instruction *instr,
7263                                struct instruction_data *data,
7264                                uint32_t n_instr)
7265 {
7266         uint32_t i;
7267
7268         for (i = 1; i < n_instr; i++) {
7269                 instr[0].type++;
7270                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7271                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7272                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7273                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7274
7275                 data[i].invalid = 1;
7276         }
7277 }
7278
7279 static uint32_t
7280 instr_pattern_dma_many_optimize(struct instruction *instructions,
7281                struct instruction_data *instruction_data,
7282                uint32_t n_instructions)
7283 {
7284         uint32_t i;
7285
7286         for (i = 0; i < n_instructions; ) {
7287                 struct instruction *instr = &instructions[i];
7288                 struct instruction_data *data = &instruction_data[i];
7289                 uint32_t n_instr = 0;
7290                 int detected;
7291
7292                 /* DMA many. */
7293                 detected = instr_pattern_dma_many_search(instr,
7294                                                          data,
7295                                                          n_instructions - i,
7296                                                          &n_instr);
7297                 if (detected) {
7298                         instr_pattern_dma_many_replace(instr, data, n_instr);
7299                         i += n_instr;
7300                         continue;
7301                 }
7302
7303                 /* No pattern starting at the current instruction. */
7304                 i++;
7305         }
7306
7307         /* Eliminate the invalid instructions that have been optimized out. */
7308         n_instructions = instr_compact(instructions,
7309                                        instruction_data,
7310                                        n_instructions);
7311
7312         return n_instructions;
7313 }
7314
7315 static uint32_t
7316 instr_optimize(struct rte_swx_pipeline *p,
7317                struct action *a,
7318                struct instruction *instructions,
7319                struct instruction_data *instruction_data,
7320                uint32_t n_instructions)
7321 {
7322         /* Extract many. */
7323         n_instructions = instr_pattern_extract_many_optimize(instructions,
7324                                                              instruction_data,
7325                                                              n_instructions);
7326
7327         /* Emit many + TX. */
7328         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7329                                                              instruction_data,
7330                                                              n_instructions);
7331
7332         /* Mov all + validate. */
7333         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7334                                                                  a,
7335                                                                  instructions,
7336                                                                  instruction_data,
7337                                                                  n_instructions);
7338
7339         /* DMA many. */
7340         n_instructions = instr_pattern_dma_many_optimize(instructions,
7341                                                          instruction_data,
7342                                                          n_instructions);
7343
7344         return n_instructions;
7345 }
7346
7347 static int
7348 instruction_config(struct rte_swx_pipeline *p,
7349                    struct action *a,
7350                    const char **instructions,
7351                    uint32_t n_instructions)
7352 {
7353         struct instruction *instr = NULL;
7354         struct instruction_data *data = NULL;
7355         int err = 0;
7356         uint32_t i;
7357
7358         CHECK(n_instructions, EINVAL);
7359         CHECK(instructions, EINVAL);
7360         for (i = 0; i < n_instructions; i++)
7361                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7362
7363         /* Memory allocation. */
7364         instr = calloc(n_instructions, sizeof(struct instruction));
7365         if (!instr) {
7366                 err = -ENOMEM;
7367                 goto error;
7368         }
7369
7370         data = calloc(n_instructions, sizeof(struct instruction_data));
7371         if (!data) {
7372                 err = -ENOMEM;
7373                 goto error;
7374         }
7375
7376         for (i = 0; i < n_instructions; i++) {
7377                 char *string = strdup(instructions[i]);
7378                 if (!string) {
7379                         err = -ENOMEM;
7380                         goto error;
7381                 }
7382
7383                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7384                 if (err) {
7385                         free(string);
7386                         goto error;
7387                 }
7388
7389                 free(string);
7390         }
7391
7392         err = instr_label_check(data, n_instructions);
7393         if (err)
7394                 goto error;
7395
7396         err = instr_verify(p, a, instr, data, n_instructions);
7397         if (err)
7398                 goto error;
7399
7400         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7401
7402         err = instr_jmp_resolve(instr, data, n_instructions);
7403         if (err)
7404                 goto error;
7405
7406         if (a) {
7407                 a->instructions = instr;
7408                 a->n_instructions = n_instructions;
7409         } else {
7410                 p->instructions = instr;
7411                 p->n_instructions = n_instructions;
7412         }
7413
7414         free(data);
7415         return 0;
7416
7417 error:
7418         free(data);
7419         free(instr);
7420         return err;
7421 }
7422
7423 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7424
7425 static instr_exec_t instruction_table[] = {
7426         [INSTR_RX] = instr_rx_exec,
7427         [INSTR_TX] = instr_tx_exec,
7428         [INSTR_TX_I] = instr_tx_i_exec,
7429
7430         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7431         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7432         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7433         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7434         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7435         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7436         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7437         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7438         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7439         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7440
7441         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7442         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7443         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7444         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7445         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7446         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7447         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7448         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7449         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7450
7451         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7452         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7453
7454         [INSTR_MOV] = instr_mov_exec,
7455         [INSTR_MOV_MH] = instr_mov_mh_exec,
7456         [INSTR_MOV_HM] = instr_mov_hm_exec,
7457         [INSTR_MOV_HH] = instr_mov_hh_exec,
7458         [INSTR_MOV_I] = instr_mov_i_exec,
7459
7460         [INSTR_DMA_HT] = instr_dma_ht_exec,
7461         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7462         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7463         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7464         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7465         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7466         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7467         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7468
7469         [INSTR_ALU_ADD] = instr_alu_add_exec,
7470         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7471         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7472         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7473         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7474         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7475
7476         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7477         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7478         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7479         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7480         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7481         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7482
7483         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7484         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7485         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7486         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7487
7488         [INSTR_ALU_AND] = instr_alu_and_exec,
7489         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7490         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7491         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7492         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7493
7494         [INSTR_ALU_OR] = instr_alu_or_exec,
7495         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7496         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7497         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7498         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7499
7500         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7501         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7502         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7503         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7504         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7505
7506         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7507         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7508         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7509         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7510         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7511         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7512
7513         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7514         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7515         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7516         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7517         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7518         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7519
7520         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7521         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7522         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7523
7524         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7525         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7526         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7527         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7528         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7529         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7530
7531         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7532         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7533         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7534         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7535         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7536         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7537         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7538         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7539         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7540
7541         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7542         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7543         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7544         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7545         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7546         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7547         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7548         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7549         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7550
7551         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7552         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7553         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7554
7555         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7556         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7557         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7558         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7559         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7560         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7561         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7562         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7563         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7564         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7565         [INSTR_METER_IMM] = instr_meter_imm_exec,
7566         [INSTR_METER_IMI] = instr_meter_imi_exec,
7567
7568         [INSTR_TABLE] = instr_table_exec,
7569         [INSTR_SELECTOR] = instr_selector_exec,
7570         [INSTR_LEARNER] = instr_learner_exec,
7571         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7572         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7573         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7574         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7575
7576         [INSTR_JMP] = instr_jmp_exec,
7577         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7578         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7579         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7580         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7581         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7582         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7583
7584         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7585         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7586         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7587         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7588         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7589
7590         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7591         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7592         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7593         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7594         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7595
7596         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7597         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7598         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7599         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7600         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7601         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7602
7603         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7604         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7605         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7606         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7607         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7608         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7609
7610         [INSTR_RETURN] = instr_return_exec,
7611 };
7612
7613 static inline void
7614 instr_exec(struct rte_swx_pipeline *p)
7615 {
7616         struct thread *t = &p->threads[p->thread_id];
7617         struct instruction *ip = t->ip;
7618         instr_exec_t instr = instruction_table[ip->type];
7619
7620         instr(p);
7621 }
7622
7623 /*
7624  * Action.
7625  */
7626 static struct action *
7627 action_find(struct rte_swx_pipeline *p, const char *name)
7628 {
7629         struct action *elem;
7630
7631         if (!name)
7632                 return NULL;
7633
7634         TAILQ_FOREACH(elem, &p->actions, node)
7635                 if (strcmp(elem->name, name) == 0)
7636                         return elem;
7637
7638         return NULL;
7639 }
7640
7641 static struct action *
7642 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7643 {
7644         struct action *action = NULL;
7645
7646         TAILQ_FOREACH(action, &p->actions, node)
7647                 if (action->id == id)
7648                         return action;
7649
7650         return NULL;
7651 }
7652
7653 static struct field *
7654 action_field_find(struct action *a, const char *name)
7655 {
7656         return a->st ? struct_type_field_find(a->st, name) : NULL;
7657 }
7658
7659 static struct field *
7660 action_field_parse(struct action *action, const char *name)
7661 {
7662         if (name[0] != 't' || name[1] != '.')
7663                 return NULL;
7664
7665         return action_field_find(action, &name[2]);
7666 }
7667
7668 static int
7669 action_has_nbo_args(struct action *a)
7670 {
7671         uint32_t i;
7672
7673         /* Return if the action does not have any args. */
7674         if (!a->st)
7675                 return 0; /* FALSE */
7676
7677         for (i = 0; i < a->st->n_fields; i++)
7678                 if (a->args_endianness[i])
7679                         return 1; /* TRUE */
7680
7681         return 0; /* FALSE */
7682 }
7683
7684 static int
7685 action_does_learning(struct action *a)
7686 {
7687         uint32_t i;
7688
7689         for (i = 0; i < a->n_instructions; i++)
7690                 switch (a->instructions[i].type) {
7691                 case INSTR_LEARNER_LEARN:
7692                         return 1; /* TRUE */
7693
7694                 case INSTR_LEARNER_FORGET:
7695                         return 1; /* TRUE */
7696
7697                 default:
7698                         continue;
7699                 }
7700
7701         return 0; /* FALSE */
7702 }
7703
7704 int
7705 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7706                                const char *name,
7707                                const char *args_struct_type_name,
7708                                const char **instructions,
7709                                uint32_t n_instructions)
7710 {
7711         struct struct_type *args_struct_type = NULL;
7712         struct action *a;
7713         int err;
7714
7715         CHECK(p, EINVAL);
7716
7717         CHECK_NAME(name, EINVAL);
7718         CHECK(!action_find(p, name), EEXIST);
7719
7720         if (args_struct_type_name) {
7721                 CHECK_NAME(args_struct_type_name, EINVAL);
7722                 args_struct_type = struct_type_find(p, args_struct_type_name);
7723                 CHECK(args_struct_type, EINVAL);
7724                 CHECK(!args_struct_type->var_size, EINVAL);
7725         }
7726
7727         /* Node allocation. */
7728         a = calloc(1, sizeof(struct action));
7729         CHECK(a, ENOMEM);
7730         if (args_struct_type) {
7731                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7732                 if (!a->args_endianness) {
7733                         free(a);
7734                         CHECK(0, ENOMEM);
7735                 }
7736         }
7737
7738         /* Node initialization. */
7739         strcpy(a->name, name);
7740         a->st = args_struct_type;
7741         a->id = p->n_actions;
7742
7743         /* Instruction translation. */
7744         err = instruction_config(p, a, instructions, n_instructions);
7745         if (err) {
7746                 free(a->args_endianness);
7747                 free(a);
7748                 return err;
7749         }
7750
7751         /* Node add to tailq. */
7752         TAILQ_INSERT_TAIL(&p->actions, a, node);
7753         p->n_actions++;
7754
7755         return 0;
7756 }
7757
7758 static int
7759 action_build(struct rte_swx_pipeline *p)
7760 {
7761         struct action *action;
7762
7763         p->action_instructions = calloc(p->n_actions,
7764                                         sizeof(struct instruction *));
7765         CHECK(p->action_instructions, ENOMEM);
7766
7767         TAILQ_FOREACH(action, &p->actions, node)
7768                 p->action_instructions[action->id] = action->instructions;
7769
7770         return 0;
7771 }
7772
7773 static void
7774 action_build_free(struct rte_swx_pipeline *p)
7775 {
7776         free(p->action_instructions);
7777         p->action_instructions = NULL;
7778 }
7779
7780 static void
7781 action_free(struct rte_swx_pipeline *p)
7782 {
7783         action_build_free(p);
7784
7785         for ( ; ; ) {
7786                 struct action *action;
7787
7788                 action = TAILQ_FIRST(&p->actions);
7789                 if (!action)
7790                         break;
7791
7792                 TAILQ_REMOVE(&p->actions, action, node);
7793                 free(action->instructions);
7794                 free(action);
7795         }
7796 }
7797
7798 static uint32_t
7799 action_arg_src_mov_count(struct action *a,
7800                          uint32_t arg_id,
7801                          struct instruction *instructions,
7802                          struct instruction_data *instruction_data,
7803                          uint32_t n_instructions)
7804 {
7805         uint32_t offset, n_users = 0, i;
7806
7807         if (!a->st ||
7808             (arg_id >= a->st->n_fields) ||
7809             !instructions ||
7810             !instruction_data ||
7811             !n_instructions)
7812                 return 0;
7813
7814         offset = a->st->fields[arg_id].offset / 8;
7815
7816         for (i = 0; i < n_instructions; i++) {
7817                 struct instruction *instr = &instructions[i];
7818                 struct instruction_data *data = &instruction_data[i];
7819
7820                 if (data->invalid ||
7821                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7822                     instr->mov.src.struct_id ||
7823                     (instr->mov.src.offset != offset))
7824                         continue;
7825
7826                 n_users++;
7827         }
7828
7829         return n_users;
7830 }
7831
7832 /*
7833  * Table.
7834  */
7835 static struct table_type *
7836 table_type_find(struct rte_swx_pipeline *p, const char *name)
7837 {
7838         struct table_type *elem;
7839
7840         TAILQ_FOREACH(elem, &p->table_types, node)
7841                 if (strcmp(elem->name, name) == 0)
7842                         return elem;
7843
7844         return NULL;
7845 }
7846
7847 static struct table_type *
7848 table_type_resolve(struct rte_swx_pipeline *p,
7849                    const char *recommended_type_name,
7850                    enum rte_swx_table_match_type match_type)
7851 {
7852         struct table_type *elem;
7853
7854         /* Only consider the recommended type if the match type is correct. */
7855         if (recommended_type_name)
7856                 TAILQ_FOREACH(elem, &p->table_types, node)
7857                         if (!strcmp(elem->name, recommended_type_name) &&
7858                             (elem->match_type == match_type))
7859                                 return elem;
7860
7861         /* Ignore the recommended type and get the first element with this match
7862          * type.
7863          */
7864         TAILQ_FOREACH(elem, &p->table_types, node)
7865                 if (elem->match_type == match_type)
7866                         return elem;
7867
7868         return NULL;
7869 }
7870
7871 static struct table *
7872 table_find(struct rte_swx_pipeline *p, const char *name)
7873 {
7874         struct table *elem;
7875
7876         TAILQ_FOREACH(elem, &p->tables, node)
7877                 if (strcmp(elem->name, name) == 0)
7878                         return elem;
7879
7880         return NULL;
7881 }
7882
7883 static struct table *
7884 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7885 {
7886         struct table *table = NULL;
7887
7888         TAILQ_FOREACH(table, &p->tables, node)
7889                 if (table->id == id)
7890                         return table;
7891
7892         return NULL;
7893 }
7894
7895 int
7896 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7897                                      const char *name,
7898                                      enum rte_swx_table_match_type match_type,
7899                                      struct rte_swx_table_ops *ops)
7900 {
7901         struct table_type *elem;
7902
7903         CHECK(p, EINVAL);
7904
7905         CHECK_NAME(name, EINVAL);
7906         CHECK(!table_type_find(p, name), EEXIST);
7907
7908         CHECK(ops, EINVAL);
7909         CHECK(ops->create, EINVAL);
7910         CHECK(ops->lkp, EINVAL);
7911         CHECK(ops->free, EINVAL);
7912
7913         /* Node allocation. */
7914         elem = calloc(1, sizeof(struct table_type));
7915         CHECK(elem, ENOMEM);
7916
7917         /* Node initialization. */
7918         strcpy(elem->name, name);
7919         elem->match_type = match_type;
7920         memcpy(&elem->ops, ops, sizeof(*ops));
7921
7922         /* Node add to tailq. */
7923         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7924
7925         return 0;
7926 }
7927
7928 static int
7929 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7930                          uint32_t n_fields,
7931                          enum rte_swx_table_match_type *match_type)
7932 {
7933         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7934
7935         for (i = 0; i < n_fields; i++) {
7936                 struct rte_swx_match_field_params  *f = &fields[i];
7937
7938                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7939                         n_fields_em++;
7940
7941                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7942                         n_fields_lpm++;
7943         }
7944
7945         if ((n_fields_lpm > 1) ||
7946             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7947                 return -EINVAL;
7948
7949         *match_type = (n_fields_em == n_fields) ?
7950                        RTE_SWX_TABLE_MATCH_EXACT :
7951                        RTE_SWX_TABLE_MATCH_WILDCARD;
7952
7953         return 0;
7954 }
7955
7956 static int
7957 table_match_fields_check(struct rte_swx_pipeline *p,
7958                          struct rte_swx_pipeline_table_params *params,
7959                          struct header **header)
7960 {
7961         struct header *h0 = NULL;
7962         struct field *hf, *mf;
7963         uint32_t *offset = NULL, i;
7964         int status = 0;
7965
7966         /* Return if no match fields. */
7967         if (!params->n_fields) {
7968                 if (params->fields) {
7969                         status = -EINVAL;
7970                         goto end;
7971                 }
7972
7973                 if (header)
7974                         *header = NULL;
7975
7976                 return 0;
7977         }
7978
7979         /* Memory allocation. */
7980         offset = calloc(params->n_fields, sizeof(uint32_t));
7981         if (!offset) {
7982                 status = -ENOMEM;
7983                 goto end;
7984         }
7985
7986         /* Check that all the match fields belong to either the same header or
7987          * to the meta-data.
7988          */
7989         hf = header_field_parse(p, params->fields[0].name, &h0);
7990         mf = metadata_field_parse(p, params->fields[0].name);
7991         if ((!hf && !mf) || (hf && hf->var_size)) {
7992                 status = -EINVAL;
7993                 goto end;
7994         }
7995
7996         offset[0] = h0 ? hf->offset : mf->offset;
7997
7998         for (i = 1; i < params->n_fields; i++)
7999                 if (h0) {
8000                         struct header *h;
8001
8002                         hf = header_field_parse(p, params->fields[i].name, &h);
8003                         if (!hf || (h->id != h0->id) || hf->var_size) {
8004                                 status = -EINVAL;
8005                                 goto end;
8006                         }
8007
8008                         offset[i] = hf->offset;
8009                 } else {
8010                         mf = metadata_field_parse(p, params->fields[i].name);
8011                         if (!mf) {
8012                                 status = -EINVAL;
8013                                 goto end;
8014                         }
8015
8016                         offset[i] = mf->offset;
8017                 }
8018
8019         /* Check that there are no duplicated match fields. */
8020         for (i = 0; i < params->n_fields; i++) {
8021                 uint32_t j;
8022
8023                 for (j = 0; j < i; j++)
8024                         if (offset[j] == offset[i]) {
8025                                 status = -EINVAL;
8026                                 goto end;
8027                         }
8028         }
8029
8030         /* Return. */
8031         if (header)
8032                 *header = h0;
8033
8034 end:
8035         free(offset);
8036         return status;
8037 }
8038
8039 int
8040 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8041                               const char *name,
8042                               struct rte_swx_pipeline_table_params *params,
8043                               const char *recommended_table_type_name,
8044                               const char *args,
8045                               uint32_t size)
8046 {
8047         struct table_type *type;
8048         struct table *t;
8049         struct action *default_action;
8050         struct header *header = NULL;
8051         uint32_t action_data_size_max = 0, i;
8052         int status = 0;
8053
8054         CHECK(p, EINVAL);
8055
8056         CHECK_NAME(name, EINVAL);
8057         CHECK(!table_find(p, name), EEXIST);
8058         CHECK(!selector_find(p, name), EEXIST);
8059         CHECK(!learner_find(p, name), EEXIST);
8060
8061         CHECK(params, EINVAL);
8062
8063         /* Match checks. */
8064         status = table_match_fields_check(p, params, &header);
8065         if (status)
8066                 return status;
8067
8068         /* Action checks. */
8069         CHECK(params->n_actions, EINVAL);
8070         CHECK(params->action_names, EINVAL);
8071         for (i = 0; i < params->n_actions; i++) {
8072                 const char *action_name = params->action_names[i];
8073                 struct action *a;
8074                 uint32_t action_data_size;
8075
8076                 CHECK_NAME(action_name, EINVAL);
8077
8078                 a = action_find(p, action_name);
8079                 CHECK(a, EINVAL);
8080                 CHECK(!action_does_learning(a), EINVAL);
8081
8082                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8083                 if (action_data_size > action_data_size_max)
8084                         action_data_size_max = action_data_size;
8085         }
8086
8087         CHECK_NAME(params->default_action_name, EINVAL);
8088         for (i = 0; i < p->n_actions; i++)
8089                 if (!strcmp(params->action_names[i],
8090                             params->default_action_name))
8091                         break;
8092         CHECK(i < params->n_actions, EINVAL);
8093         default_action = action_find(p, params->default_action_name);
8094         CHECK((default_action->st && params->default_action_data) ||
8095               !params->default_action_data, EINVAL);
8096
8097         /* Table type checks. */
8098         if (recommended_table_type_name)
8099                 CHECK_NAME(recommended_table_type_name, EINVAL);
8100
8101         if (params->n_fields) {
8102                 enum rte_swx_table_match_type match_type;
8103
8104                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8105                 if (status)
8106                         return status;
8107
8108                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8109                 CHECK(type, EINVAL);
8110         } else {
8111                 type = NULL;
8112         }
8113
8114         /* Memory allocation. */
8115         t = calloc(1, sizeof(struct table));
8116         CHECK(t, ENOMEM);
8117
8118         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8119         if (!t->fields) {
8120                 free(t);
8121                 CHECK(0, ENOMEM);
8122         }
8123
8124         t->actions = calloc(params->n_actions, sizeof(struct action *));
8125         if (!t->actions) {
8126                 free(t->fields);
8127                 free(t);
8128                 CHECK(0, ENOMEM);
8129         }
8130
8131         if (action_data_size_max) {
8132                 t->default_action_data = calloc(1, action_data_size_max);
8133                 if (!t->default_action_data) {
8134                         free(t->actions);
8135                         free(t->fields);
8136                         free(t);
8137                         CHECK(0, ENOMEM);
8138                 }
8139         }
8140
8141         /* Node initialization. */
8142         strcpy(t->name, name);
8143         if (args && args[0])
8144                 strcpy(t->args, args);
8145         t->type = type;
8146
8147         for (i = 0; i < params->n_fields; i++) {
8148                 struct rte_swx_match_field_params *field = &params->fields[i];
8149                 struct match_field *f = &t->fields[i];
8150
8151                 f->match_type = field->match_type;
8152                 f->field = header ?
8153                         header_field_parse(p, field->name, NULL) :
8154                         metadata_field_parse(p, field->name);
8155         }
8156         t->n_fields = params->n_fields;
8157         t->header = header;
8158
8159         for (i = 0; i < params->n_actions; i++)
8160                 t->actions[i] = action_find(p, params->action_names[i]);
8161         t->default_action = default_action;
8162         if (default_action->st)
8163                 memcpy(t->default_action_data,
8164                        params->default_action_data,
8165                        default_action->st->n_bits / 8);
8166         t->n_actions = params->n_actions;
8167         t->default_action_is_const = params->default_action_is_const;
8168         t->action_data_size_max = action_data_size_max;
8169
8170         t->size = size;
8171         t->id = p->n_tables;
8172
8173         /* Node add to tailq. */
8174         TAILQ_INSERT_TAIL(&p->tables, t, node);
8175         p->n_tables++;
8176
8177         return 0;
8178 }
8179
8180 static struct rte_swx_table_params *
8181 table_params_get(struct table *table)
8182 {
8183         struct rte_swx_table_params *params;
8184         struct field *first, *last;
8185         uint8_t *key_mask;
8186         uint32_t key_size, key_offset, action_data_size, i;
8187
8188         /* Memory allocation. */
8189         params = calloc(1, sizeof(struct rte_swx_table_params));
8190         if (!params)
8191                 return NULL;
8192
8193         /* Find first (smallest offset) and last (biggest offset) match fields. */
8194         first = table->fields[0].field;
8195         last = table->fields[0].field;
8196
8197         for (i = 0; i < table->n_fields; i++) {
8198                 struct field *f = table->fields[i].field;
8199
8200                 if (f->offset < first->offset)
8201                         first = f;
8202
8203                 if (f->offset > last->offset)
8204                         last = f;
8205         }
8206
8207         /* Key offset and size. */
8208         key_offset = first->offset / 8;
8209         key_size = (last->offset + last->n_bits - first->offset) / 8;
8210
8211         /* Memory allocation. */
8212         key_mask = calloc(1, key_size);
8213         if (!key_mask) {
8214                 free(params);
8215                 return NULL;
8216         }
8217
8218         /* Key mask. */
8219         for (i = 0; i < table->n_fields; i++) {
8220                 struct field *f = table->fields[i].field;
8221                 uint32_t start = (f->offset - first->offset) / 8;
8222                 size_t size = f->n_bits / 8;
8223
8224                 memset(&key_mask[start], 0xFF, size);
8225         }
8226
8227         /* Action data size. */
8228         action_data_size = 0;
8229         for (i = 0; i < table->n_actions; i++) {
8230                 struct action *action = table->actions[i];
8231                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8232
8233                 if (ads > action_data_size)
8234                         action_data_size = ads;
8235         }
8236
8237         /* Fill in. */
8238         params->match_type = table->type->match_type;
8239         params->key_size = key_size;
8240         params->key_offset = key_offset;
8241         params->key_mask0 = key_mask;
8242         params->action_data_size = action_data_size;
8243         params->n_keys_max = table->size;
8244
8245         return params;
8246 }
8247
8248 static void
8249 table_params_free(struct rte_swx_table_params *params)
8250 {
8251         if (!params)
8252                 return;
8253
8254         free(params->key_mask0);
8255         free(params);
8256 }
8257
8258 static int
8259 table_stub_lkp(void *table __rte_unused,
8260                void *mailbox __rte_unused,
8261                uint8_t **key __rte_unused,
8262                uint64_t *action_id __rte_unused,
8263                uint8_t **action_data __rte_unused,
8264                int *hit)
8265 {
8266         *hit = 0;
8267         return 1; /* DONE. */
8268 }
8269
8270 static int
8271 table_build(struct rte_swx_pipeline *p)
8272 {
8273         uint32_t i;
8274
8275         /* Per pipeline: table statistics. */
8276         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8277         CHECK(p->table_stats, ENOMEM);
8278
8279         for (i = 0; i < p->n_tables; i++) {
8280                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8281                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8282         }
8283
8284         /* Per thread: table runt-time. */
8285         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8286                 struct thread *t = &p->threads[i];
8287                 struct table *table;
8288
8289                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8290                 CHECK(t->tables, ENOMEM);
8291
8292                 TAILQ_FOREACH(table, &p->tables, node) {
8293                         struct table_runtime *r = &t->tables[table->id];
8294
8295                         if (table->type) {
8296                                 uint64_t size;
8297
8298                                 size = table->type->ops.mailbox_size_get();
8299
8300                                 /* r->func. */
8301                                 r->func = table->type->ops.lkp;
8302
8303                                 /* r->mailbox. */
8304                                 if (size) {
8305                                         r->mailbox = calloc(1, size);
8306                                         CHECK(r->mailbox, ENOMEM);
8307                                 }
8308
8309                                 /* r->key. */
8310                                 r->key = table->header ?
8311                                         &t->structs[table->header->struct_id] :
8312                                         &t->structs[p->metadata_struct_id];
8313                         } else {
8314                                 r->func = table_stub_lkp;
8315                         }
8316                 }
8317         }
8318
8319         return 0;
8320 }
8321
8322 static void
8323 table_build_free(struct rte_swx_pipeline *p)
8324 {
8325         uint32_t i;
8326
8327         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8328                 struct thread *t = &p->threads[i];
8329                 uint32_t j;
8330
8331                 if (!t->tables)
8332                         continue;
8333
8334                 for (j = 0; j < p->n_tables; j++) {
8335                         struct table_runtime *r = &t->tables[j];
8336
8337                         free(r->mailbox);
8338                 }
8339
8340                 free(t->tables);
8341                 t->tables = NULL;
8342         }
8343
8344         if (p->table_stats) {
8345                 for (i = 0; i < p->n_tables; i++)
8346                         free(p->table_stats[i].n_pkts_action);
8347
8348                 free(p->table_stats);
8349         }
8350 }
8351
8352 static void
8353 table_free(struct rte_swx_pipeline *p)
8354 {
8355         table_build_free(p);
8356
8357         /* Tables. */
8358         for ( ; ; ) {
8359                 struct table *elem;
8360
8361                 elem = TAILQ_FIRST(&p->tables);
8362                 if (!elem)
8363                         break;
8364
8365                 TAILQ_REMOVE(&p->tables, elem, node);
8366                 free(elem->fields);
8367                 free(elem->actions);
8368                 free(elem->default_action_data);
8369                 free(elem);
8370         }
8371
8372         /* Table types. */
8373         for ( ; ; ) {
8374                 struct table_type *elem;
8375
8376                 elem = TAILQ_FIRST(&p->table_types);
8377                 if (!elem)
8378                         break;
8379
8380                 TAILQ_REMOVE(&p->table_types, elem, node);
8381                 free(elem);
8382         }
8383 }
8384
8385 /*
8386  * Selector.
8387  */
8388 static struct selector *
8389 selector_find(struct rte_swx_pipeline *p, const char *name)
8390 {
8391         struct selector *s;
8392
8393         TAILQ_FOREACH(s, &p->selectors, node)
8394                 if (strcmp(s->name, name) == 0)
8395                         return s;
8396
8397         return NULL;
8398 }
8399
8400 static struct selector *
8401 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8402 {
8403         struct selector *s = NULL;
8404
8405         TAILQ_FOREACH(s, &p->selectors, node)
8406                 if (s->id == id)
8407                         return s;
8408
8409         return NULL;
8410 }
8411
8412 static int
8413 selector_fields_check(struct rte_swx_pipeline *p,
8414                       struct rte_swx_pipeline_selector_params *params,
8415                       struct header **header)
8416 {
8417         struct header *h0 = NULL;
8418         struct field *hf, *mf;
8419         uint32_t i;
8420
8421         /* Return if no selector fields. */
8422         if (!params->n_selector_fields || !params->selector_field_names)
8423                 return -EINVAL;
8424
8425         /* Check that all the selector fields either belong to the same header
8426          * or are all meta-data fields.
8427          */
8428         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8429         mf = metadata_field_parse(p, params->selector_field_names[0]);
8430         if (!hf && !mf)
8431                 return -EINVAL;
8432
8433         for (i = 1; i < params->n_selector_fields; i++)
8434                 if (h0) {
8435                         struct header *h;
8436
8437                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8438                         if (!hf || (h->id != h0->id))
8439                                 return -EINVAL;
8440                 } else {
8441                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8442                         if (!mf)
8443                                 return -EINVAL;
8444                 }
8445
8446         /* Check that there are no duplicated match fields. */
8447         for (i = 0; i < params->n_selector_fields; i++) {
8448                 const char *field_name = params->selector_field_names[i];
8449                 uint32_t j;
8450
8451                 for (j = i + 1; j < params->n_selector_fields; j++)
8452                         if (!strcmp(params->selector_field_names[j], field_name))
8453                                 return -EINVAL;
8454         }
8455
8456         /* Return. */
8457         if (header)
8458                 *header = h0;
8459
8460         return 0;
8461 }
8462
8463 int
8464 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8465                                  const char *name,
8466                                  struct rte_swx_pipeline_selector_params *params)
8467 {
8468         struct selector *s;
8469         struct header *selector_header = NULL;
8470         struct field *group_id_field, *member_id_field;
8471         uint32_t i;
8472         int status = 0;
8473
8474         CHECK(p, EINVAL);
8475
8476         CHECK_NAME(name, EINVAL);
8477         CHECK(!table_find(p, name), EEXIST);
8478         CHECK(!selector_find(p, name), EEXIST);
8479         CHECK(!learner_find(p, name), EEXIST);
8480
8481         CHECK(params, EINVAL);
8482
8483         CHECK_NAME(params->group_id_field_name, EINVAL);
8484         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8485         CHECK(group_id_field, EINVAL);
8486
8487         for (i = 0; i < params->n_selector_fields; i++) {
8488                 const char *field_name = params->selector_field_names[i];
8489
8490                 CHECK_NAME(field_name, EINVAL);
8491         }
8492         status = selector_fields_check(p, params, &selector_header);
8493         if (status)
8494                 return status;
8495
8496         CHECK_NAME(params->member_id_field_name, EINVAL);
8497         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8498         CHECK(member_id_field, EINVAL);
8499
8500         CHECK(params->n_groups_max, EINVAL);
8501
8502         CHECK(params->n_members_per_group_max, EINVAL);
8503
8504         /* Memory allocation. */
8505         s = calloc(1, sizeof(struct selector));
8506         if (!s) {
8507                 status = -ENOMEM;
8508                 goto error;
8509         }
8510
8511         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8512         if (!s->selector_fields) {
8513                 status = -ENOMEM;
8514                 goto error;
8515         }
8516
8517         /* Node initialization. */
8518         strcpy(s->name, name);
8519
8520         s->group_id_field = group_id_field;
8521
8522         for (i = 0; i < params->n_selector_fields; i++) {
8523                 const char *field_name = params->selector_field_names[i];
8524
8525                 s->selector_fields[i] = selector_header ?
8526                         header_field_parse(p, field_name, NULL) :
8527                         metadata_field_parse(p, field_name);
8528         }
8529
8530         s->n_selector_fields = params->n_selector_fields;
8531
8532         s->selector_header = selector_header;
8533
8534         s->member_id_field = member_id_field;
8535
8536         s->n_groups_max = params->n_groups_max;
8537
8538         s->n_members_per_group_max = params->n_members_per_group_max;
8539
8540         s->id = p->n_selectors;
8541
8542         /* Node add to tailq. */
8543         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8544         p->n_selectors++;
8545
8546         return 0;
8547
8548 error:
8549         if (!s)
8550                 return status;
8551
8552         free(s->selector_fields);
8553
8554         free(s);
8555
8556         return status;
8557 }
8558
8559 static void
8560 selector_params_free(struct rte_swx_table_selector_params *params)
8561 {
8562         if (!params)
8563                 return;
8564
8565         free(params->selector_mask);
8566
8567         free(params);
8568 }
8569
8570 static struct rte_swx_table_selector_params *
8571 selector_table_params_get(struct selector *s)
8572 {
8573         struct rte_swx_table_selector_params *params = NULL;
8574         struct field *first, *last;
8575         uint32_t i;
8576
8577         /* Memory allocation. */
8578         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8579         if (!params)
8580                 goto error;
8581
8582         /* Group ID. */
8583         params->group_id_offset = s->group_id_field->offset / 8;
8584
8585         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8586         first = s->selector_fields[0];
8587         last = s->selector_fields[0];
8588
8589         for (i = 0; i < s->n_selector_fields; i++) {
8590                 struct field *f = s->selector_fields[i];
8591
8592                 if (f->offset < first->offset)
8593                         first = f;
8594
8595                 if (f->offset > last->offset)
8596                         last = f;
8597         }
8598
8599         /* Selector offset and size. */
8600         params->selector_offset = first->offset / 8;
8601         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8602
8603         /* Memory allocation. */
8604         params->selector_mask = calloc(1, params->selector_size);
8605         if (!params->selector_mask)
8606                 goto error;
8607
8608         /* Selector mask. */
8609         for (i = 0; i < s->n_selector_fields; i++) {
8610                 struct field *f = s->selector_fields[i];
8611                 uint32_t start = (f->offset - first->offset) / 8;
8612                 size_t size = f->n_bits / 8;
8613
8614                 memset(&params->selector_mask[start], 0xFF, size);
8615         }
8616
8617         /* Member ID. */
8618         params->member_id_offset = s->member_id_field->offset / 8;
8619
8620         /* Maximum number of groups. */
8621         params->n_groups_max = s->n_groups_max;
8622
8623         /* Maximum number of members per group. */
8624         params->n_members_per_group_max = s->n_members_per_group_max;
8625
8626         return params;
8627
8628 error:
8629         selector_params_free(params);
8630         return NULL;
8631 }
8632
8633 static void
8634 selector_build_free(struct rte_swx_pipeline *p)
8635 {
8636         uint32_t i;
8637
8638         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8639                 struct thread *t = &p->threads[i];
8640                 uint32_t j;
8641
8642                 if (!t->selectors)
8643                         continue;
8644
8645                 for (j = 0; j < p->n_selectors; j++) {
8646                         struct selector_runtime *r = &t->selectors[j];
8647
8648                         free(r->mailbox);
8649                 }
8650
8651                 free(t->selectors);
8652                 t->selectors = NULL;
8653         }
8654
8655         free(p->selector_stats);
8656         p->selector_stats = NULL;
8657 }
8658
8659 static int
8660 selector_build(struct rte_swx_pipeline *p)
8661 {
8662         uint32_t i;
8663         int status = 0;
8664
8665         /* Per pipeline: selector statistics. */
8666         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8667         if (!p->selector_stats) {
8668                 status = -ENOMEM;
8669                 goto error;
8670         }
8671
8672         /* Per thread: selector run-time. */
8673         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8674                 struct thread *t = &p->threads[i];
8675                 struct selector *s;
8676
8677                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8678                 if (!t->selectors) {
8679                         status = -ENOMEM;
8680                         goto error;
8681                 }
8682
8683                 TAILQ_FOREACH(s, &p->selectors, node) {
8684                         struct selector_runtime *r = &t->selectors[s->id];
8685                         uint64_t size;
8686
8687                         /* r->mailbox. */
8688                         size = rte_swx_table_selector_mailbox_size_get();
8689                         if (size) {
8690                                 r->mailbox = calloc(1, size);
8691                                 if (!r->mailbox) {
8692                                         status = -ENOMEM;
8693                                         goto error;
8694                                 }
8695                         }
8696
8697                         /* r->group_id_buffer. */
8698                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8699
8700                         /* r->selector_buffer. */
8701                         r->selector_buffer = s->selector_header ?
8702                                 &t->structs[s->selector_header->struct_id] :
8703                                 &t->structs[p->metadata_struct_id];
8704
8705                         /* r->member_id_buffer. */
8706                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8707                 }
8708         }
8709
8710         return 0;
8711
8712 error:
8713         selector_build_free(p);
8714         return status;
8715 }
8716
8717 static void
8718 selector_free(struct rte_swx_pipeline *p)
8719 {
8720         selector_build_free(p);
8721
8722         /* Selector tables. */
8723         for ( ; ; ) {
8724                 struct selector *elem;
8725
8726                 elem = TAILQ_FIRST(&p->selectors);
8727                 if (!elem)
8728                         break;
8729
8730                 TAILQ_REMOVE(&p->selectors, elem, node);
8731                 free(elem->selector_fields);
8732                 free(elem);
8733         }
8734 }
8735
8736 /*
8737  * Learner table.
8738  */
8739 static struct learner *
8740 learner_find(struct rte_swx_pipeline *p, const char *name)
8741 {
8742         struct learner *l;
8743
8744         TAILQ_FOREACH(l, &p->learners, node)
8745                 if (!strcmp(l->name, name))
8746                         return l;
8747
8748         return NULL;
8749 }
8750
8751 static struct learner *
8752 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8753 {
8754         struct learner *l = NULL;
8755
8756         TAILQ_FOREACH(l, &p->learners, node)
8757                 if (l->id == id)
8758                         return l;
8759
8760         return NULL;
8761 }
8762
8763 static int
8764 learner_match_fields_check(struct rte_swx_pipeline *p,
8765                            struct rte_swx_pipeline_learner_params *params,
8766                            struct header **header)
8767 {
8768         struct header *h0 = NULL;
8769         struct field *hf, *mf;
8770         uint32_t i;
8771
8772         /* Return if no match fields. */
8773         if (!params->n_fields || !params->field_names)
8774                 return -EINVAL;
8775
8776         /* Check that all the match fields either belong to the same header
8777          * or are all meta-data fields.
8778          */
8779         hf = header_field_parse(p, params->field_names[0], &h0);
8780         mf = metadata_field_parse(p, params->field_names[0]);
8781         if (!hf && !mf)
8782                 return -EINVAL;
8783
8784         for (i = 1; i < params->n_fields; i++)
8785                 if (h0) {
8786                         struct header *h;
8787
8788                         hf = header_field_parse(p, params->field_names[i], &h);
8789                         if (!hf || (h->id != h0->id))
8790                                 return -EINVAL;
8791                 } else {
8792                         mf = metadata_field_parse(p, params->field_names[i]);
8793                         if (!mf)
8794                                 return -EINVAL;
8795                 }
8796
8797         /* Check that there are no duplicated match fields. */
8798         for (i = 0; i < params->n_fields; i++) {
8799                 const char *field_name = params->field_names[i];
8800                 uint32_t j;
8801
8802                 for (j = i + 1; j < params->n_fields; j++)
8803                         if (!strcmp(params->field_names[j], field_name))
8804                                 return -EINVAL;
8805         }
8806
8807         /* Return. */
8808         if (header)
8809                 *header = h0;
8810
8811         return 0;
8812 }
8813
8814 static int
8815 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8816 {
8817         struct struct_type *mst = p->metadata_st, *ast = a->st;
8818         struct field *mf, *af;
8819         uint32_t mf_pos, i;
8820
8821         if (!ast) {
8822                 if (mf_name)
8823                         return -EINVAL;
8824
8825                 return 0;
8826         }
8827
8828         /* Check that mf_name is the name of a valid meta-data field. */
8829         CHECK_NAME(mf_name, EINVAL);
8830         mf = metadata_field_parse(p, mf_name);
8831         CHECK(mf, EINVAL);
8832
8833         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8834          * all the action arguments.
8835          */
8836         mf_pos = mf - mst->fields;
8837         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8838
8839         /* Check that the size of each of the identified meta-data fields matches exactly the size
8840          * of the corresponding action argument.
8841          */
8842         for (i = 0; i < ast->n_fields; i++) {
8843                 mf = &mst->fields[mf_pos + i];
8844                 af = &ast->fields[i];
8845
8846                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8847         }
8848
8849         return 0;
8850 }
8851
8852 static int
8853 learner_action_learning_check(struct rte_swx_pipeline *p,
8854                               struct action *action,
8855                               const char **action_names,
8856                               uint32_t n_actions)
8857 {
8858         uint32_t i;
8859
8860         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8861          * the action passed as argument to the "learn" instruction) is also enabled for the
8862          * current learner table.
8863          */
8864         for (i = 0; i < action->n_instructions; i++) {
8865                 struct instruction *instr = &action->instructions[i];
8866                 uint32_t found = 0, j;
8867
8868                 if (instr->type != INSTR_LEARNER_LEARN)
8869                         continue;
8870
8871                 for (j = 0; j < n_actions; j++) {
8872                         struct action *a;
8873
8874                         a = action_find(p, action_names[j]);
8875                         if (!a)
8876                                 return -EINVAL;
8877
8878                         if (a->id == instr->learn.action_id)
8879                                 found = 1;
8880                 }
8881
8882                 if (!found)
8883                         return -EINVAL;
8884         }
8885
8886         return 0;
8887 }
8888
8889 int
8890 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8891                               const char *name,
8892                               struct rte_swx_pipeline_learner_params *params,
8893                               uint32_t size,
8894                               uint32_t timeout)
8895 {
8896         struct learner *l = NULL;
8897         struct action *default_action;
8898         struct header *header = NULL;
8899         uint32_t action_data_size_max = 0, i;
8900         int status = 0;
8901
8902         CHECK(p, EINVAL);
8903
8904         CHECK_NAME(name, EINVAL);
8905         CHECK(!table_find(p, name), EEXIST);
8906         CHECK(!selector_find(p, name), EEXIST);
8907         CHECK(!learner_find(p, name), EEXIST);
8908
8909         CHECK(params, EINVAL);
8910
8911         /* Match checks. */
8912         status = learner_match_fields_check(p, params, &header);
8913         if (status)
8914                 return status;
8915
8916         /* Action checks. */
8917         CHECK(params->n_actions, EINVAL);
8918
8919         CHECK(params->action_names, EINVAL);
8920         for (i = 0; i < params->n_actions; i++) {
8921                 const char *action_name = params->action_names[i];
8922                 const char *action_field_name = params->action_field_names[i];
8923                 struct action *a;
8924                 uint32_t action_data_size;
8925
8926                 CHECK_NAME(action_name, EINVAL);
8927
8928                 a = action_find(p, action_name);
8929                 CHECK(a, EINVAL);
8930
8931                 status = learner_action_args_check(p, a, action_field_name);
8932                 if (status)
8933                         return status;
8934
8935                 status = learner_action_learning_check(p,
8936                                                        a,
8937                                                        params->action_names,
8938                                                        params->n_actions);
8939                 if (status)
8940                         return status;
8941
8942                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8943                 if (action_data_size > action_data_size_max)
8944                         action_data_size_max = action_data_size;
8945         }
8946
8947         CHECK_NAME(params->default_action_name, EINVAL);
8948         for (i = 0; i < p->n_actions; i++)
8949                 if (!strcmp(params->action_names[i],
8950                             params->default_action_name))
8951                         break;
8952         CHECK(i < params->n_actions, EINVAL);
8953
8954         default_action = action_find(p, params->default_action_name);
8955         CHECK((default_action->st && params->default_action_data) ||
8956               !params->default_action_data, EINVAL);
8957
8958         /* Any other checks. */
8959         CHECK(size, EINVAL);
8960         CHECK(timeout, EINVAL);
8961
8962         /* Memory allocation. */
8963         l = calloc(1, sizeof(struct learner));
8964         if (!l)
8965                 goto nomem;
8966
8967         l->fields = calloc(params->n_fields, sizeof(struct field *));
8968         if (!l->fields)
8969                 goto nomem;
8970
8971         l->actions = calloc(params->n_actions, sizeof(struct action *));
8972         if (!l->actions)
8973                 goto nomem;
8974
8975         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8976         if (!l->action_arg)
8977                 goto nomem;
8978
8979         if (action_data_size_max) {
8980                 l->default_action_data = calloc(1, action_data_size_max);
8981                 if (!l->default_action_data)
8982                         goto nomem;
8983         }
8984
8985         /* Node initialization. */
8986         strcpy(l->name, name);
8987
8988         for (i = 0; i < params->n_fields; i++) {
8989                 const char *field_name = params->field_names[i];
8990
8991                 l->fields[i] = header ?
8992                         header_field_parse(p, field_name, NULL) :
8993                         metadata_field_parse(p, field_name);
8994         }
8995
8996         l->n_fields = params->n_fields;
8997
8998         l->header = header;
8999
9000         for (i = 0; i < params->n_actions; i++) {
9001                 const char *mf_name = params->action_field_names[i];
9002
9003                 l->actions[i] = action_find(p, params->action_names[i]);
9004
9005                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9006         }
9007
9008         l->default_action = default_action;
9009
9010         if (default_action->st)
9011                 memcpy(l->default_action_data,
9012                        params->default_action_data,
9013                        default_action->st->n_bits / 8);
9014
9015         l->n_actions = params->n_actions;
9016
9017         l->default_action_is_const = params->default_action_is_const;
9018
9019         l->action_data_size_max = action_data_size_max;
9020
9021         l->size = size;
9022
9023         l->timeout = timeout;
9024
9025         l->id = p->n_learners;
9026
9027         /* Node add to tailq. */
9028         TAILQ_INSERT_TAIL(&p->learners, l, node);
9029         p->n_learners++;
9030
9031         return 0;
9032
9033 nomem:
9034         if (!l)
9035                 return -ENOMEM;
9036
9037         free(l->action_arg);
9038         free(l->actions);
9039         free(l->fields);
9040         free(l);
9041
9042         return -ENOMEM;
9043 }
9044
9045 static void
9046 learner_params_free(struct rte_swx_table_learner_params *params)
9047 {
9048         if (!params)
9049                 return;
9050
9051         free(params->key_mask0);
9052
9053         free(params);
9054 }
9055
9056 static struct rte_swx_table_learner_params *
9057 learner_params_get(struct learner *l)
9058 {
9059         struct rte_swx_table_learner_params *params = NULL;
9060         struct field *first, *last;
9061         uint32_t i;
9062
9063         /* Memory allocation. */
9064         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9065         if (!params)
9066                 goto error;
9067
9068         /* Find first (smallest offset) and last (biggest offset) match fields. */
9069         first = l->fields[0];
9070         last = l->fields[0];
9071
9072         for (i = 0; i < l->n_fields; i++) {
9073                 struct field *f = l->fields[i];
9074
9075                 if (f->offset < first->offset)
9076                         first = f;
9077
9078                 if (f->offset > last->offset)
9079                         last = f;
9080         }
9081
9082         /* Key offset and size. */
9083         params->key_offset = first->offset / 8;
9084         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9085
9086         /* Memory allocation. */
9087         params->key_mask0 = calloc(1, params->key_size);
9088         if (!params->key_mask0)
9089                 goto error;
9090
9091         /* Key mask. */
9092         for (i = 0; i < l->n_fields; i++) {
9093                 struct field *f = l->fields[i];
9094                 uint32_t start = (f->offset - first->offset) / 8;
9095                 size_t size = f->n_bits / 8;
9096
9097                 memset(&params->key_mask0[start], 0xFF, size);
9098         }
9099
9100         /* Action data size. */
9101         params->action_data_size = l->action_data_size_max;
9102
9103         /* Maximum number of keys. */
9104         params->n_keys_max = l->size;
9105
9106         /* Timeout. */
9107         params->key_timeout = l->timeout;
9108
9109         return params;
9110
9111 error:
9112         learner_params_free(params);
9113         return NULL;
9114 }
9115
9116 static void
9117 learner_build_free(struct rte_swx_pipeline *p)
9118 {
9119         uint32_t i;
9120
9121         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9122                 struct thread *t = &p->threads[i];
9123                 uint32_t j;
9124
9125                 if (!t->learners)
9126                         continue;
9127
9128                 for (j = 0; j < p->n_learners; j++) {
9129                         struct learner_runtime *r = &t->learners[j];
9130
9131                         free(r->mailbox);
9132                         free(r->action_data);
9133                 }
9134
9135                 free(t->learners);
9136                 t->learners = NULL;
9137         }
9138
9139         if (p->learner_stats) {
9140                 for (i = 0; i < p->n_learners; i++)
9141                         free(p->learner_stats[i].n_pkts_action);
9142
9143                 free(p->learner_stats);
9144         }
9145 }
9146
9147 static int
9148 learner_build(struct rte_swx_pipeline *p)
9149 {
9150         uint32_t i;
9151         int status = 0;
9152
9153         /* Per pipeline: learner statistics. */
9154         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9155         CHECK(p->learner_stats, ENOMEM);
9156
9157         for (i = 0; i < p->n_learners; i++) {
9158                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9159                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9160         }
9161
9162         /* Per thread: learner run-time. */
9163         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9164                 struct thread *t = &p->threads[i];
9165                 struct learner *l;
9166
9167                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9168                 if (!t->learners) {
9169                         status = -ENOMEM;
9170                         goto error;
9171                 }
9172
9173                 TAILQ_FOREACH(l, &p->learners, node) {
9174                         struct learner_runtime *r = &t->learners[l->id];
9175                         uint64_t size;
9176                         uint32_t j;
9177
9178                         /* r->mailbox. */
9179                         size = rte_swx_table_learner_mailbox_size_get();
9180                         if (size) {
9181                                 r->mailbox = calloc(1, size);
9182                                 if (!r->mailbox) {
9183                                         status = -ENOMEM;
9184                                         goto error;
9185                                 }
9186                         }
9187
9188                         /* r->key. */
9189                         r->key = l->header ?
9190                                 &t->structs[l->header->struct_id] :
9191                                 &t->structs[p->metadata_struct_id];
9192
9193                         /* r->action_data. */
9194                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9195                         if (!r->action_data) {
9196                                 status = -ENOMEM;
9197                                 goto error;
9198                         }
9199
9200                         for (j = 0; j < l->n_actions; j++) {
9201                                 struct action *a = l->actions[j];
9202                                 struct field *mf = l->action_arg[j];
9203                                 uint8_t *m = t->structs[p->metadata_struct_id];
9204
9205                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9206                         }
9207                 }
9208         }
9209
9210         return 0;
9211
9212 error:
9213         learner_build_free(p);
9214         return status;
9215 }
9216
9217 static void
9218 learner_free(struct rte_swx_pipeline *p)
9219 {
9220         learner_build_free(p);
9221
9222         /* Learner tables. */
9223         for ( ; ; ) {
9224                 struct learner *l;
9225
9226                 l = TAILQ_FIRST(&p->learners);
9227                 if (!l)
9228                         break;
9229
9230                 TAILQ_REMOVE(&p->learners, l, node);
9231                 free(l->fields);
9232                 free(l->actions);
9233                 free(l->action_arg);
9234                 free(l->default_action_data);
9235                 free(l);
9236         }
9237 }
9238
9239 /*
9240  * Table state.
9241  */
9242 static int
9243 table_state_build(struct rte_swx_pipeline *p)
9244 {
9245         struct table *table;
9246         struct selector *s;
9247         struct learner *l;
9248
9249         p->table_state = calloc(p->n_tables + p->n_selectors,
9250                                 sizeof(struct rte_swx_table_state));
9251         CHECK(p->table_state, ENOMEM);
9252
9253         TAILQ_FOREACH(table, &p->tables, node) {
9254                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9255
9256                 if (table->type) {
9257                         struct rte_swx_table_params *params;
9258
9259                         /* ts->obj. */
9260                         params = table_params_get(table);
9261                         CHECK(params, ENOMEM);
9262
9263                         ts->obj = table->type->ops.create(params,
9264                                 NULL,
9265                                 table->args,
9266                                 p->numa_node);
9267
9268                         table_params_free(params);
9269                         CHECK(ts->obj, ENODEV);
9270                 }
9271
9272                 /* ts->default_action_data. */
9273                 if (table->action_data_size_max) {
9274                         ts->default_action_data =
9275                                 malloc(table->action_data_size_max);
9276                         CHECK(ts->default_action_data, ENOMEM);
9277
9278                         memcpy(ts->default_action_data,
9279                                table->default_action_data,
9280                                table->action_data_size_max);
9281                 }
9282
9283                 /* ts->default_action_id. */
9284                 ts->default_action_id = table->default_action->id;
9285         }
9286
9287         TAILQ_FOREACH(s, &p->selectors, node) {
9288                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9289                 struct rte_swx_table_selector_params *params;
9290
9291                 /* ts->obj. */
9292                 params = selector_table_params_get(s);
9293                 CHECK(params, ENOMEM);
9294
9295                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9296
9297                 selector_params_free(params);
9298                 CHECK(ts->obj, ENODEV);
9299         }
9300
9301         TAILQ_FOREACH(l, &p->learners, node) {
9302                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9303                         p->n_selectors + l->id];
9304                 struct rte_swx_table_learner_params *params;
9305
9306                 /* ts->obj. */
9307                 params = learner_params_get(l);
9308                 CHECK(params, ENOMEM);
9309
9310                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9311                 learner_params_free(params);
9312                 CHECK(ts->obj, ENODEV);
9313
9314                 /* ts->default_action_data. */
9315                 if (l->action_data_size_max) {
9316                         ts->default_action_data = malloc(l->action_data_size_max);
9317                         CHECK(ts->default_action_data, ENOMEM);
9318
9319                         memcpy(ts->default_action_data,
9320                                l->default_action_data,
9321                                l->action_data_size_max);
9322                 }
9323
9324                 /* ts->default_action_id. */
9325                 ts->default_action_id = l->default_action->id;
9326         }
9327
9328         return 0;
9329 }
9330
9331 static void
9332 table_state_build_free(struct rte_swx_pipeline *p)
9333 {
9334         uint32_t i;
9335
9336         if (!p->table_state)
9337                 return;
9338
9339         for (i = 0; i < p->n_tables; i++) {
9340                 struct rte_swx_table_state *ts = &p->table_state[i];
9341                 struct table *table = table_find_by_id(p, i);
9342
9343                 /* ts->obj. */
9344                 if (table->type && ts->obj)
9345                         table->type->ops.free(ts->obj);
9346
9347                 /* ts->default_action_data. */
9348                 free(ts->default_action_data);
9349         }
9350
9351         for (i = 0; i < p->n_selectors; i++) {
9352                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9353
9354                 /* ts->obj. */
9355                 if (ts->obj)
9356                         rte_swx_table_selector_free(ts->obj);
9357         }
9358
9359         for (i = 0; i < p->n_learners; i++) {
9360                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9361
9362                 /* ts->obj. */
9363                 if (ts->obj)
9364                         rte_swx_table_learner_free(ts->obj);
9365
9366                 /* ts->default_action_data. */
9367                 free(ts->default_action_data);
9368         }
9369
9370         free(p->table_state);
9371         p->table_state = NULL;
9372 }
9373
9374 static void
9375 table_state_free(struct rte_swx_pipeline *p)
9376 {
9377         table_state_build_free(p);
9378 }
9379
9380 /*
9381  * Register array.
9382  */
9383 static struct regarray *
9384 regarray_find(struct rte_swx_pipeline *p, const char *name)
9385 {
9386         struct regarray *elem;
9387
9388         TAILQ_FOREACH(elem, &p->regarrays, node)
9389                 if (!strcmp(elem->name, name))
9390                         return elem;
9391
9392         return NULL;
9393 }
9394
9395 static struct regarray *
9396 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9397 {
9398         struct regarray *elem = NULL;
9399
9400         TAILQ_FOREACH(elem, &p->regarrays, node)
9401                 if (elem->id == id)
9402                         return elem;
9403
9404         return NULL;
9405 }
9406
9407 int
9408 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9409                               const char *name,
9410                               uint32_t size,
9411                               uint64_t init_val)
9412 {
9413         struct regarray *r;
9414
9415         CHECK(p, EINVAL);
9416
9417         CHECK_NAME(name, EINVAL);
9418         CHECK(!regarray_find(p, name), EEXIST);
9419
9420         CHECK(size, EINVAL);
9421         size = rte_align32pow2(size);
9422
9423         /* Memory allocation. */
9424         r = calloc(1, sizeof(struct regarray));
9425         CHECK(r, ENOMEM);
9426
9427         /* Node initialization. */
9428         strcpy(r->name, name);
9429         r->init_val = init_val;
9430         r->size = size;
9431         r->id = p->n_regarrays;
9432
9433         /* Node add to tailq. */
9434         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9435         p->n_regarrays++;
9436
9437         return 0;
9438 }
9439
9440 static int
9441 regarray_build(struct rte_swx_pipeline *p)
9442 {
9443         struct regarray *regarray;
9444
9445         if (!p->n_regarrays)
9446                 return 0;
9447
9448         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9449         CHECK(p->regarray_runtime, ENOMEM);
9450
9451         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9452                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9453                 uint32_t i;
9454
9455                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9456                                          RTE_CACHE_LINE_SIZE,
9457                                          p->numa_node);
9458                 CHECK(r->regarray, ENOMEM);
9459
9460                 if (regarray->init_val)
9461                         for (i = 0; i < regarray->size; i++)
9462                                 r->regarray[i] = regarray->init_val;
9463
9464                 r->size_mask = regarray->size - 1;
9465         }
9466
9467         return 0;
9468 }
9469
9470 static void
9471 regarray_build_free(struct rte_swx_pipeline *p)
9472 {
9473         uint32_t i;
9474
9475         if (!p->regarray_runtime)
9476                 return;
9477
9478         for (i = 0; i < p->n_regarrays; i++) {
9479                 struct regarray *regarray = regarray_find_by_id(p, i);
9480                 struct regarray_runtime *r = &p->regarray_runtime[i];
9481
9482                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9483         }
9484
9485         free(p->regarray_runtime);
9486         p->regarray_runtime = NULL;
9487 }
9488
9489 static void
9490 regarray_free(struct rte_swx_pipeline *p)
9491 {
9492         regarray_build_free(p);
9493
9494         for ( ; ; ) {
9495                 struct regarray *elem;
9496
9497                 elem = TAILQ_FIRST(&p->regarrays);
9498                 if (!elem)
9499                         break;
9500
9501                 TAILQ_REMOVE(&p->regarrays, elem, node);
9502                 free(elem);
9503         }
9504 }
9505
9506 /*
9507  * Meter array.
9508  */
9509 static struct meter_profile *
9510 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9511 {
9512         struct meter_profile *elem;
9513
9514         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9515                 if (!strcmp(elem->name, name))
9516                         return elem;
9517
9518         return NULL;
9519 }
9520
9521 static struct metarray *
9522 metarray_find(struct rte_swx_pipeline *p, const char *name)
9523 {
9524         struct metarray *elem;
9525
9526         TAILQ_FOREACH(elem, &p->metarrays, node)
9527                 if (!strcmp(elem->name, name))
9528                         return elem;
9529
9530         return NULL;
9531 }
9532
9533 static struct metarray *
9534 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9535 {
9536         struct metarray *elem = NULL;
9537
9538         TAILQ_FOREACH(elem, &p->metarrays, node)
9539                 if (elem->id == id)
9540                         return elem;
9541
9542         return NULL;
9543 }
9544
9545 int
9546 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9547                                  const char *name,
9548                                  uint32_t size)
9549 {
9550         struct metarray *m;
9551
9552         CHECK(p, EINVAL);
9553
9554         CHECK_NAME(name, EINVAL);
9555         CHECK(!metarray_find(p, name), EEXIST);
9556
9557         CHECK(size, EINVAL);
9558         size = rte_align32pow2(size);
9559
9560         /* Memory allocation. */
9561         m = calloc(1, sizeof(struct metarray));
9562         CHECK(m, ENOMEM);
9563
9564         /* Node initialization. */
9565         strcpy(m->name, name);
9566         m->size = size;
9567         m->id = p->n_metarrays;
9568
9569         /* Node add to tailq. */
9570         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9571         p->n_metarrays++;
9572
9573         return 0;
9574 }
9575
9576 struct meter_profile meter_profile_default = {
9577         .node = {0},
9578         .name = "",
9579         .params = {0},
9580
9581         .profile = {
9582                 .cbs = 10000,
9583                 .pbs = 10000,
9584                 .cir_period = 1,
9585                 .cir_bytes_per_period = 1,
9586                 .pir_period = 1,
9587                 .pir_bytes_per_period = 1,
9588         },
9589
9590         .n_users = 0,
9591 };
9592
9593 static void
9594 meter_init(struct meter *m)
9595 {
9596         memset(m, 0, sizeof(struct meter));
9597         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9598         m->profile = &meter_profile_default;
9599         m->color_mask = RTE_COLOR_GREEN;
9600
9601         meter_profile_default.n_users++;
9602 }
9603
9604 static int
9605 metarray_build(struct rte_swx_pipeline *p)
9606 {
9607         struct metarray *m;
9608
9609         if (!p->n_metarrays)
9610                 return 0;
9611
9612         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9613         CHECK(p->metarray_runtime, ENOMEM);
9614
9615         TAILQ_FOREACH(m, &p->metarrays, node) {
9616                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9617                 uint32_t i;
9618
9619                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9620                                          RTE_CACHE_LINE_SIZE,
9621                                          p->numa_node);
9622                 CHECK(r->metarray, ENOMEM);
9623
9624                 for (i = 0; i < m->size; i++)
9625                         meter_init(&r->metarray[i]);
9626
9627                 r->size_mask = m->size - 1;
9628         }
9629
9630         return 0;
9631 }
9632
9633 static void
9634 metarray_build_free(struct rte_swx_pipeline *p)
9635 {
9636         uint32_t i;
9637
9638         if (!p->metarray_runtime)
9639                 return;
9640
9641         for (i = 0; i < p->n_metarrays; i++) {
9642                 struct metarray *m = metarray_find_by_id(p, i);
9643                 struct metarray_runtime *r = &p->metarray_runtime[i];
9644
9645                 env_free(r->metarray, m->size * sizeof(struct meter));
9646         }
9647
9648         free(p->metarray_runtime);
9649         p->metarray_runtime = NULL;
9650 }
9651
9652 static void
9653 metarray_free(struct rte_swx_pipeline *p)
9654 {
9655         metarray_build_free(p);
9656
9657         /* Meter arrays. */
9658         for ( ; ; ) {
9659                 struct metarray *elem;
9660
9661                 elem = TAILQ_FIRST(&p->metarrays);
9662                 if (!elem)
9663                         break;
9664
9665                 TAILQ_REMOVE(&p->metarrays, elem, node);
9666                 free(elem);
9667         }
9668
9669         /* Meter profiles. */
9670         for ( ; ; ) {
9671                 struct meter_profile *elem;
9672
9673                 elem = TAILQ_FIRST(&p->meter_profiles);
9674                 if (!elem)
9675                         break;
9676
9677                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9678                 free(elem);
9679         }
9680 }
9681
9682 /*
9683  * Pipeline.
9684  */
9685 int
9686 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9687 {
9688         struct rte_swx_pipeline *pipeline;
9689
9690         /* Check input parameters. */
9691         CHECK(p, EINVAL);
9692
9693         /* Memory allocation. */
9694         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9695         CHECK(pipeline, ENOMEM);
9696
9697         /* Initialization. */
9698         TAILQ_INIT(&pipeline->struct_types);
9699         TAILQ_INIT(&pipeline->port_in_types);
9700         TAILQ_INIT(&pipeline->ports_in);
9701         TAILQ_INIT(&pipeline->port_out_types);
9702         TAILQ_INIT(&pipeline->ports_out);
9703         TAILQ_INIT(&pipeline->extern_types);
9704         TAILQ_INIT(&pipeline->extern_objs);
9705         TAILQ_INIT(&pipeline->extern_funcs);
9706         TAILQ_INIT(&pipeline->headers);
9707         TAILQ_INIT(&pipeline->actions);
9708         TAILQ_INIT(&pipeline->table_types);
9709         TAILQ_INIT(&pipeline->tables);
9710         TAILQ_INIT(&pipeline->selectors);
9711         TAILQ_INIT(&pipeline->learners);
9712         TAILQ_INIT(&pipeline->regarrays);
9713         TAILQ_INIT(&pipeline->meter_profiles);
9714         TAILQ_INIT(&pipeline->metarrays);
9715
9716         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9717         pipeline->numa_node = numa_node;
9718
9719         *p = pipeline;
9720         return 0;
9721 }
9722
9723 void
9724 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9725 {
9726         if (!p)
9727                 return;
9728
9729         free(p->instructions);
9730
9731         metarray_free(p);
9732         regarray_free(p);
9733         table_state_free(p);
9734         learner_free(p);
9735         selector_free(p);
9736         table_free(p);
9737         action_free(p);
9738         metadata_free(p);
9739         header_free(p);
9740         extern_func_free(p);
9741         extern_obj_free(p);
9742         port_out_free(p);
9743         port_in_free(p);
9744         struct_free(p);
9745
9746         free(p);
9747 }
9748
9749 int
9750 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9751                                      const char **instructions,
9752                                      uint32_t n_instructions)
9753 {
9754         int err;
9755         uint32_t i;
9756
9757         err = instruction_config(p, NULL, instructions, n_instructions);
9758         if (err)
9759                 return err;
9760
9761         /* Thread instruction pointer reset. */
9762         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9763                 struct thread *t = &p->threads[i];
9764
9765                 thread_ip_reset(p, t);
9766         }
9767
9768         return 0;
9769 }
9770
9771 int
9772 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9773 {
9774         int status;
9775
9776         CHECK(p, EINVAL);
9777         CHECK(p->build_done == 0, EEXIST);
9778
9779         status = port_in_build(p);
9780         if (status)
9781                 goto error;
9782
9783         status = port_out_build(p);
9784         if (status)
9785                 goto error;
9786
9787         status = struct_build(p);
9788         if (status)
9789                 goto error;
9790
9791         status = extern_obj_build(p);
9792         if (status)
9793                 goto error;
9794
9795         status = extern_func_build(p);
9796         if (status)
9797                 goto error;
9798
9799         status = header_build(p);
9800         if (status)
9801                 goto error;
9802
9803         status = metadata_build(p);
9804         if (status)
9805                 goto error;
9806
9807         status = action_build(p);
9808         if (status)
9809                 goto error;
9810
9811         status = table_build(p);
9812         if (status)
9813                 goto error;
9814
9815         status = selector_build(p);
9816         if (status)
9817                 goto error;
9818
9819         status = learner_build(p);
9820         if (status)
9821                 goto error;
9822
9823         status = table_state_build(p);
9824         if (status)
9825                 goto error;
9826
9827         status = regarray_build(p);
9828         if (status)
9829                 goto error;
9830
9831         status = metarray_build(p);
9832         if (status)
9833                 goto error;
9834
9835         p->build_done = 1;
9836         return 0;
9837
9838 error:
9839         metarray_build_free(p);
9840         regarray_build_free(p);
9841         table_state_build_free(p);
9842         learner_build_free(p);
9843         selector_build_free(p);
9844         table_build_free(p);
9845         action_build_free(p);
9846         metadata_build_free(p);
9847         header_build_free(p);
9848         extern_func_build_free(p);
9849         extern_obj_build_free(p);
9850         port_out_build_free(p);
9851         port_in_build_free(p);
9852         struct_build_free(p);
9853
9854         return status;
9855 }
9856
9857 void
9858 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9859 {
9860         uint32_t i;
9861
9862         for (i = 0; i < n_instructions; i++)
9863                 instr_exec(p);
9864 }
9865
9866 void
9867 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9868 {
9869         uint32_t i;
9870
9871         for (i = 0; i < p->n_ports_out; i++) {
9872                 struct port_out_runtime *port = &p->out[i];
9873
9874                 if (port->flush)
9875                         port->flush(port->obj);
9876         }
9877 }
9878
9879 /*
9880  * Control.
9881  */
9882 int
9883 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9884                               struct rte_swx_ctl_pipeline_info *pipeline)
9885 {
9886         struct action *action;
9887         struct table *table;
9888         uint32_t n_actions = 0, n_tables = 0;
9889
9890         if (!p || !pipeline)
9891                 return -EINVAL;
9892
9893         TAILQ_FOREACH(action, &p->actions, node)
9894                 n_actions++;
9895
9896         TAILQ_FOREACH(table, &p->tables, node)
9897                 n_tables++;
9898
9899         pipeline->n_ports_in = p->n_ports_in;
9900         pipeline->n_ports_out = p->n_ports_out;
9901         pipeline->n_actions = n_actions;
9902         pipeline->n_tables = n_tables;
9903         pipeline->n_selectors = p->n_selectors;
9904         pipeline->n_learners = p->n_learners;
9905         pipeline->n_regarrays = p->n_regarrays;
9906         pipeline->n_metarrays = p->n_metarrays;
9907
9908         return 0;
9909 }
9910
9911 int
9912 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9913 {
9914         if (!p || !numa_node)
9915                 return -EINVAL;
9916
9917         *numa_node = p->numa_node;
9918         return 0;
9919 }
9920
9921 int
9922 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9923                             uint32_t action_id,
9924                             struct rte_swx_ctl_action_info *action)
9925 {
9926         struct action *a = NULL;
9927
9928         if (!p || (action_id >= p->n_actions) || !action)
9929                 return -EINVAL;
9930
9931         a = action_find_by_id(p, action_id);
9932         if (!a)
9933                 return -EINVAL;
9934
9935         strcpy(action->name, a->name);
9936         action->n_args = a->st ? a->st->n_fields : 0;
9937         return 0;
9938 }
9939
9940 int
9941 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9942                                 uint32_t action_id,
9943                                 uint32_t action_arg_id,
9944                                 struct rte_swx_ctl_action_arg_info *action_arg)
9945 {
9946         struct action *a = NULL;
9947         struct field *arg = NULL;
9948
9949         if (!p || (action_id >= p->n_actions) || !action_arg)
9950                 return -EINVAL;
9951
9952         a = action_find_by_id(p, action_id);
9953         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9954                 return -EINVAL;
9955
9956         arg = &a->st->fields[action_arg_id];
9957         strcpy(action_arg->name, arg->name);
9958         action_arg->n_bits = arg->n_bits;
9959         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9960
9961         return 0;
9962 }
9963
9964 int
9965 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9966                            uint32_t table_id,
9967                            struct rte_swx_ctl_table_info *table)
9968 {
9969         struct table *t = NULL;
9970
9971         if (!p || !table)
9972                 return -EINVAL;
9973
9974         t = table_find_by_id(p, table_id);
9975         if (!t)
9976                 return -EINVAL;
9977
9978         strcpy(table->name, t->name);
9979         strcpy(table->args, t->args);
9980         table->n_match_fields = t->n_fields;
9981         table->n_actions = t->n_actions;
9982         table->default_action_is_const = t->default_action_is_const;
9983         table->size = t->size;
9984         return 0;
9985 }
9986
9987 int
9988 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9989         uint32_t table_id,
9990         uint32_t match_field_id,
9991         struct rte_swx_ctl_table_match_field_info *match_field)
9992 {
9993         struct table *t;
9994         struct match_field *f;
9995
9996         if (!p || (table_id >= p->n_tables) || !match_field)
9997                 return -EINVAL;
9998
9999         t = table_find_by_id(p, table_id);
10000         if (!t || (match_field_id >= t->n_fields))
10001                 return -EINVAL;
10002
10003         f = &t->fields[match_field_id];
10004         match_field->match_type = f->match_type;
10005         match_field->is_header = t->header ? 1 : 0;
10006         match_field->n_bits = f->field->n_bits;
10007         match_field->offset = f->field->offset;
10008
10009         return 0;
10010 }
10011
10012 int
10013 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10014         uint32_t table_id,
10015         uint32_t table_action_id,
10016         struct rte_swx_ctl_table_action_info *table_action)
10017 {
10018         struct table *t;
10019
10020         if (!p || (table_id >= p->n_tables) || !table_action)
10021                 return -EINVAL;
10022
10023         t = table_find_by_id(p, table_id);
10024         if (!t || (table_action_id >= t->n_actions))
10025                 return -EINVAL;
10026
10027         table_action->action_id = t->actions[table_action_id]->id;
10028
10029         return 0;
10030 }
10031
10032 int
10033 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10034                           uint32_t table_id,
10035                           struct rte_swx_table_ops *table_ops,
10036                           int *is_stub)
10037 {
10038         struct table *t;
10039
10040         if (!p || (table_id >= p->n_tables))
10041                 return -EINVAL;
10042
10043         t = table_find_by_id(p, table_id);
10044         if (!t)
10045                 return -EINVAL;
10046
10047         if (t->type) {
10048                 if (table_ops)
10049                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10050                 *is_stub = 0;
10051         } else {
10052                 *is_stub = 1;
10053         }
10054
10055         return 0;
10056 }
10057
10058 int
10059 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10060                               uint32_t selector_id,
10061                               struct rte_swx_ctl_selector_info *selector)
10062 {
10063         struct selector *s = NULL;
10064
10065         if (!p || !selector)
10066                 return -EINVAL;
10067
10068         s = selector_find_by_id(p, selector_id);
10069         if (!s)
10070                 return -EINVAL;
10071
10072         strcpy(selector->name, s->name);
10073
10074         selector->n_selector_fields = s->n_selector_fields;
10075         selector->n_groups_max = s->n_groups_max;
10076         selector->n_members_per_group_max = s->n_members_per_group_max;
10077
10078         return 0;
10079 }
10080
10081 int
10082 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10083          uint32_t selector_id,
10084          struct rte_swx_ctl_table_match_field_info *field)
10085 {
10086         struct selector *s;
10087
10088         if (!p || (selector_id >= p->n_selectors) || !field)
10089                 return -EINVAL;
10090
10091         s = selector_find_by_id(p, selector_id);
10092         if (!s)
10093                 return -EINVAL;
10094
10095         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10096         field->is_header = 0;
10097         field->n_bits = s->group_id_field->n_bits;
10098         field->offset = s->group_id_field->offset;
10099
10100         return 0;
10101 }
10102
10103 int
10104 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10105          uint32_t selector_id,
10106          uint32_t selector_field_id,
10107          struct rte_swx_ctl_table_match_field_info *field)
10108 {
10109         struct selector *s;
10110         struct field *f;
10111
10112         if (!p || (selector_id >= p->n_selectors) || !field)
10113                 return -EINVAL;
10114
10115         s = selector_find_by_id(p, selector_id);
10116         if (!s || (selector_field_id >= s->n_selector_fields))
10117                 return -EINVAL;
10118
10119         f = s->selector_fields[selector_field_id];
10120         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10121         field->is_header = s->selector_header ? 1 : 0;
10122         field->n_bits = f->n_bits;
10123         field->offset = f->offset;
10124
10125         return 0;
10126 }
10127
10128 int
10129 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10130          uint32_t selector_id,
10131          struct rte_swx_ctl_table_match_field_info *field)
10132 {
10133         struct selector *s;
10134
10135         if (!p || (selector_id >= p->n_selectors) || !field)
10136                 return -EINVAL;
10137
10138         s = selector_find_by_id(p, selector_id);
10139         if (!s)
10140                 return -EINVAL;
10141
10142         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10143         field->is_header = 0;
10144         field->n_bits = s->member_id_field->n_bits;
10145         field->offset = s->member_id_field->offset;
10146
10147         return 0;
10148 }
10149
10150 int
10151 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10152                              uint32_t learner_id,
10153                              struct rte_swx_ctl_learner_info *learner)
10154 {
10155         struct learner *l = NULL;
10156
10157         if (!p || !learner)
10158                 return -EINVAL;
10159
10160         l = learner_find_by_id(p, learner_id);
10161         if (!l)
10162                 return -EINVAL;
10163
10164         strcpy(learner->name, l->name);
10165
10166         learner->n_match_fields = l->n_fields;
10167         learner->n_actions = l->n_actions;
10168         learner->default_action_is_const = l->default_action_is_const;
10169         learner->size = l->size;
10170
10171         return 0;
10172 }
10173
10174 int
10175 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10176                                          uint32_t learner_id,
10177                                          uint32_t match_field_id,
10178                                          struct rte_swx_ctl_table_match_field_info *match_field)
10179 {
10180         struct learner *l;
10181         struct field *f;
10182
10183         if (!p || (learner_id >= p->n_learners) || !match_field)
10184                 return -EINVAL;
10185
10186         l = learner_find_by_id(p, learner_id);
10187         if (!l || (match_field_id >= l->n_fields))
10188                 return -EINVAL;
10189
10190         f = l->fields[match_field_id];
10191         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10192         match_field->is_header = l->header ? 1 : 0;
10193         match_field->n_bits = f->n_bits;
10194         match_field->offset = f->offset;
10195
10196         return 0;
10197 }
10198
10199 int
10200 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10201                                     uint32_t learner_id,
10202                                     uint32_t learner_action_id,
10203                                     struct rte_swx_ctl_table_action_info *learner_action)
10204 {
10205         struct learner *l;
10206
10207         if (!p || (learner_id >= p->n_learners) || !learner_action)
10208                 return -EINVAL;
10209
10210         l = learner_find_by_id(p, learner_id);
10211         if (!l || (learner_action_id >= l->n_actions))
10212                 return -EINVAL;
10213
10214         learner_action->action_id = l->actions[learner_action_id]->id;
10215
10216         return 0;
10217 }
10218
10219 int
10220 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10221                                  struct rte_swx_table_state **table_state)
10222 {
10223         if (!p || !table_state || !p->build_done)
10224                 return -EINVAL;
10225
10226         *table_state = p->table_state;
10227         return 0;
10228 }
10229
10230 int
10231 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10232                                  struct rte_swx_table_state *table_state)
10233 {
10234         if (!p || !table_state || !p->build_done)
10235                 return -EINVAL;
10236
10237         p->table_state = table_state;
10238         return 0;
10239 }
10240
10241 int
10242 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10243                                         uint32_t port_id,
10244                                         struct rte_swx_port_in_stats *stats)
10245 {
10246         struct port_in *port;
10247
10248         if (!p || !stats)
10249                 return -EINVAL;
10250
10251         port = port_in_find(p, port_id);
10252         if (!port)
10253                 return -EINVAL;
10254
10255         port->type->ops.stats_read(port->obj, stats);
10256         return 0;
10257 }
10258
10259 int
10260 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10261                                          uint32_t port_id,
10262                                          struct rte_swx_port_out_stats *stats)
10263 {
10264         struct port_out *port;
10265
10266         if (!p || !stats)
10267                 return -EINVAL;
10268
10269         port = port_out_find(p, port_id);
10270         if (!port)
10271                 return -EINVAL;
10272
10273         port->type->ops.stats_read(port->obj, stats);
10274         return 0;
10275 }
10276
10277 int
10278 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10279                                       const char *table_name,
10280                                       struct rte_swx_table_stats *stats)
10281 {
10282         struct table *table;
10283         struct table_statistics *table_stats;
10284
10285         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10286                 return -EINVAL;
10287
10288         table = table_find(p, table_name);
10289         if (!table)
10290                 return -EINVAL;
10291
10292         table_stats = &p->table_stats[table->id];
10293
10294         memcpy(stats->n_pkts_action,
10295                table_stats->n_pkts_action,
10296                p->n_actions * sizeof(uint64_t));
10297
10298         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10299         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10300
10301         return 0;
10302 }
10303
10304 int
10305 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10306         const char *selector_name,
10307         struct rte_swx_pipeline_selector_stats *stats)
10308 {
10309         struct selector *s;
10310
10311         if (!p || !selector_name || !selector_name[0] || !stats)
10312                 return -EINVAL;
10313
10314         s = selector_find(p, selector_name);
10315         if (!s)
10316                 return -EINVAL;
10317
10318         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10319
10320         return 0;
10321 }
10322
10323 int
10324 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10325                                         const char *learner_name,
10326                                         struct rte_swx_learner_stats *stats)
10327 {
10328         struct learner *l;
10329         struct learner_statistics *learner_stats;
10330
10331         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10332                 return -EINVAL;
10333
10334         l = learner_find(p, learner_name);
10335         if (!l)
10336                 return -EINVAL;
10337
10338         learner_stats = &p->learner_stats[l->id];
10339
10340         memcpy(stats->n_pkts_action,
10341                learner_stats->n_pkts_action,
10342                p->n_actions * sizeof(uint64_t));
10343
10344         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10345         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10346
10347         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10348         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10349
10350         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10351
10352         return 0;
10353 }
10354
10355 int
10356 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10357                               uint32_t regarray_id,
10358                               struct rte_swx_ctl_regarray_info *regarray)
10359 {
10360         struct regarray *r;
10361
10362         if (!p || !regarray)
10363                 return -EINVAL;
10364
10365         r = regarray_find_by_id(p, regarray_id);
10366         if (!r)
10367                 return -EINVAL;
10368
10369         strcpy(regarray->name, r->name);
10370         regarray->size = r->size;
10371         return 0;
10372 }
10373
10374 int
10375 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10376                                    const char *regarray_name,
10377                                    uint32_t regarray_index,
10378                                    uint64_t *value)
10379 {
10380         struct regarray *regarray;
10381         struct regarray_runtime *r;
10382
10383         if (!p || !regarray_name || !value)
10384                 return -EINVAL;
10385
10386         regarray = regarray_find(p, regarray_name);
10387         if (!regarray || (regarray_index >= regarray->size))
10388                 return -EINVAL;
10389
10390         r = &p->regarray_runtime[regarray->id];
10391         *value = r->regarray[regarray_index];
10392         return 0;
10393 }
10394
10395 int
10396 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10397                                    const char *regarray_name,
10398                                    uint32_t regarray_index,
10399                                    uint64_t value)
10400 {
10401         struct regarray *regarray;
10402         struct regarray_runtime *r;
10403
10404         if (!p || !regarray_name)
10405                 return -EINVAL;
10406
10407         regarray = regarray_find(p, regarray_name);
10408         if (!regarray || (regarray_index >= regarray->size))
10409                 return -EINVAL;
10410
10411         r = &p->regarray_runtime[regarray->id];
10412         r->regarray[regarray_index] = value;
10413         return 0;
10414 }
10415
10416 int
10417 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10418                               uint32_t metarray_id,
10419                               struct rte_swx_ctl_metarray_info *metarray)
10420 {
10421         struct metarray *m;
10422
10423         if (!p || !metarray)
10424                 return -EINVAL;
10425
10426         m = metarray_find_by_id(p, metarray_id);
10427         if (!m)
10428                 return -EINVAL;
10429
10430         strcpy(metarray->name, m->name);
10431         metarray->size = m->size;
10432         return 0;
10433 }
10434
10435 int
10436 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10437                               const char *name,
10438                               struct rte_meter_trtcm_params *params)
10439 {
10440         struct meter_profile *mp;
10441         int status;
10442
10443         CHECK(p, EINVAL);
10444         CHECK_NAME(name, EINVAL);
10445         CHECK(params, EINVAL);
10446         CHECK(!meter_profile_find(p, name), EEXIST);
10447
10448         /* Node allocation. */
10449         mp = calloc(1, sizeof(struct meter_profile));
10450         CHECK(mp, ENOMEM);
10451
10452         /* Node initialization. */
10453         strcpy(mp->name, name);
10454         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10455         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10456         if (status) {
10457                 free(mp);
10458                 CHECK(0, EINVAL);
10459         }
10460
10461         /* Node add to tailq. */
10462         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10463
10464         return 0;
10465 }
10466
10467 int
10468 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10469                                  const char *name)
10470 {
10471         struct meter_profile *mp;
10472
10473         CHECK(p, EINVAL);
10474         CHECK_NAME(name, EINVAL);
10475
10476         mp = meter_profile_find(p, name);
10477         CHECK(mp, EINVAL);
10478         CHECK(!mp->n_users, EBUSY);
10479
10480         /* Remove node from tailq. */
10481         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10482         free(mp);
10483
10484         return 0;
10485 }
10486
10487 int
10488 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10489                         const char *metarray_name,
10490                         uint32_t metarray_index)
10491 {
10492         struct meter_profile *mp_old;
10493         struct metarray *metarray;
10494         struct metarray_runtime *metarray_runtime;
10495         struct meter *m;
10496
10497         CHECK(p, EINVAL);
10498         CHECK_NAME(metarray_name, EINVAL);
10499
10500         metarray = metarray_find(p, metarray_name);
10501         CHECK(metarray, EINVAL);
10502         CHECK(metarray_index < metarray->size, EINVAL);
10503
10504         metarray_runtime = &p->metarray_runtime[metarray->id];
10505         m = &metarray_runtime->metarray[metarray_index];
10506         mp_old = m->profile;
10507
10508         meter_init(m);
10509
10510         mp_old->n_users--;
10511
10512         return 0;
10513 }
10514
10515 int
10516 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10517                       const char *metarray_name,
10518                       uint32_t metarray_index,
10519                       const char *profile_name)
10520 {
10521         struct meter_profile *mp, *mp_old;
10522         struct metarray *metarray;
10523         struct metarray_runtime *metarray_runtime;
10524         struct meter *m;
10525
10526         CHECK(p, EINVAL);
10527         CHECK_NAME(metarray_name, EINVAL);
10528
10529         metarray = metarray_find(p, metarray_name);
10530         CHECK(metarray, EINVAL);
10531         CHECK(metarray_index < metarray->size, EINVAL);
10532
10533         mp = meter_profile_find(p, profile_name);
10534         CHECK(mp, EINVAL);
10535
10536         metarray_runtime = &p->metarray_runtime[metarray->id];
10537         m = &metarray_runtime->metarray[metarray_index];
10538         mp_old = m->profile;
10539
10540         memset(m, 0, sizeof(struct meter));
10541         rte_meter_trtcm_config(&m->m, &mp->profile);
10542         m->profile = mp;
10543         m->color_mask = RTE_COLORS;
10544
10545         mp->n_users++;
10546         mp_old->n_users--;
10547
10548         return 0;
10549 }
10550
10551 int
10552 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10553                              const char *metarray_name,
10554                              uint32_t metarray_index,
10555                              struct rte_swx_ctl_meter_stats *stats)
10556 {
10557         struct metarray *metarray;
10558         struct metarray_runtime *metarray_runtime;
10559         struct meter *m;
10560
10561         CHECK(p, EINVAL);
10562         CHECK_NAME(metarray_name, EINVAL);
10563
10564         metarray = metarray_find(p, metarray_name);
10565         CHECK(metarray, EINVAL);
10566         CHECK(metarray_index < metarray->size, EINVAL);
10567
10568         CHECK(stats, EINVAL);
10569
10570         metarray_runtime = &p->metarray_runtime[metarray->id];
10571         m = &metarray_runtime->metarray[metarray_index];
10572
10573         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10574         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10575
10576         return 0;
10577 }