lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
1814
1815 static inline void
1816 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
1817 {
1818         struct thread *t = &p->threads[p->thread_id];
1819         struct instruction *ip = t->ip;
1820         uint64_t valid_headers = t->valid_headers;
1821         uint32_t n_headers_out = t->n_headers_out;
1822         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1823         uint8_t *ho_ptr = NULL;
1824         uint32_t ho_nbytes = 0, first = 1, i;
1825
1826         for (i = 0; i < n_emit; i++) {
1827                 uint32_t header_id = ip->io.hdr.header_id[i];
1828                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1829
1830                 struct header_runtime *hi = &t->headers[header_id];
1831                 uint8_t *hi_ptr0 = hi->ptr0;
1832                 uint32_t n_bytes = hi->n_bytes;
1833
1834                 uint8_t *hi_ptr = t->structs[struct_id];
1835
1836                 if (!MASK64_BIT_GET(valid_headers, header_id))
1837                         continue;
1838
1839                 TRACE("[Thread %2u]: emit header %u\n",
1840                       p->thread_id,
1841                       header_id);
1842
1843                 /* Headers. */
1844                 if (first) {
1845                         first = 0;
1846
1847                         if (!t->n_headers_out) {
1848                                 ho = &t->headers_out[0];
1849
1850                                 ho->ptr0 = hi_ptr0;
1851                                 ho->ptr = hi_ptr;
1852
1853                                 ho_ptr = hi_ptr;
1854                                 ho_nbytes = n_bytes;
1855
1856                                 n_headers_out = 1;
1857
1858                                 continue;
1859                         } else {
1860                                 ho_ptr = ho->ptr;
1861                                 ho_nbytes = ho->n_bytes;
1862                         }
1863                 }
1864
1865                 if (ho_ptr + ho_nbytes == hi_ptr) {
1866                         ho_nbytes += n_bytes;
1867                 } else {
1868                         ho->n_bytes = ho_nbytes;
1869
1870                         ho++;
1871                         ho->ptr0 = hi_ptr0;
1872                         ho->ptr = hi_ptr;
1873
1874                         ho_ptr = hi_ptr;
1875                         ho_nbytes = n_bytes;
1876
1877                         n_headers_out++;
1878                 }
1879         }
1880
1881         ho->n_bytes = ho_nbytes;
1882         t->n_headers_out = n_headers_out;
1883 }
1884
1885 static inline void
1886 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1887 {
1888         __instr_hdr_emit_exec(p, 1);
1889
1890         /* Thread. */
1891         thread_ip_inc(p);
1892 }
1893
1894 static inline void
1895 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1896 {
1897         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
1898               p->thread_id);
1899
1900         __instr_hdr_emit_exec(p, 1);
1901         instr_tx_exec(p);
1902 }
1903
1904 static inline void
1905 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1906 {
1907         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
1908               p->thread_id);
1909
1910         __instr_hdr_emit_exec(p, 2);
1911         instr_tx_exec(p);
1912 }
1913
1914 static inline void
1915 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1916 {
1917         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
1918               p->thread_id);
1919
1920         __instr_hdr_emit_exec(p, 3);
1921         instr_tx_exec(p);
1922 }
1923
1924 static inline void
1925 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1926 {
1927         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
1928               p->thread_id);
1929
1930         __instr_hdr_emit_exec(p, 4);
1931         instr_tx_exec(p);
1932 }
1933
1934 static inline void
1935 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1936 {
1937         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
1938               p->thread_id);
1939
1940         __instr_hdr_emit_exec(p, 5);
1941         instr_tx_exec(p);
1942 }
1943
1944 static inline void
1945 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1946 {
1947         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
1948               p->thread_id);
1949
1950         __instr_hdr_emit_exec(p, 6);
1951         instr_tx_exec(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1956 {
1957         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
1958               p->thread_id);
1959
1960         __instr_hdr_emit_exec(p, 7);
1961         instr_tx_exec(p);
1962 }
1963
1964 static inline void
1965 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1966 {
1967         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
1968               p->thread_id);
1969
1970         __instr_hdr_emit_exec(p, 8);
1971         instr_tx_exec(p);
1972 }
1973
1974 /*
1975  * validate.
1976  */
1977 static int
1978 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1979                              struct action *action __rte_unused,
1980                              char **tokens,
1981                              int n_tokens,
1982                              struct instruction *instr,
1983                              struct instruction_data *data __rte_unused)
1984 {
1985         struct header *h;
1986
1987         CHECK(n_tokens == 2, EINVAL);
1988
1989         h = header_parse(p, tokens[1]);
1990         CHECK(h, EINVAL);
1991
1992         instr->type = INSTR_HDR_VALIDATE;
1993         instr->valid.header_id = h->id;
1994         return 0;
1995 }
1996
1997 static inline void
1998 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1999 {
2000         struct thread *t = &p->threads[p->thread_id];
2001         struct instruction *ip = t->ip;
2002         uint32_t header_id = ip->valid.header_id;
2003
2004         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2005
2006         /* Headers. */
2007         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2008
2009         /* Thread. */
2010         thread_ip_inc(p);
2011 }
2012
2013 /*
2014  * invalidate.
2015  */
2016 static int
2017 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2018                                struct action *action __rte_unused,
2019                                char **tokens,
2020                                int n_tokens,
2021                                struct instruction *instr,
2022                                struct instruction_data *data __rte_unused)
2023 {
2024         struct header *h;
2025
2026         CHECK(n_tokens == 2, EINVAL);
2027
2028         h = header_parse(p, tokens[1]);
2029         CHECK(h, EINVAL);
2030
2031         instr->type = INSTR_HDR_INVALIDATE;
2032         instr->valid.header_id = h->id;
2033         return 0;
2034 }
2035
2036 static inline void
2037 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2038 {
2039         struct thread *t = &p->threads[p->thread_id];
2040         struct instruction *ip = t->ip;
2041         uint32_t header_id = ip->valid.header_id;
2042
2043         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2044
2045         /* Headers. */
2046         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2047
2048         /* Thread. */
2049         thread_ip_inc(p);
2050 }
2051
2052 /*
2053  * table.
2054  */
2055 static struct table *
2056 table_find(struct rte_swx_pipeline *p, const char *name);
2057
2058 static struct selector *
2059 selector_find(struct rte_swx_pipeline *p, const char *name);
2060
2061 static struct learner *
2062 learner_find(struct rte_swx_pipeline *p, const char *name);
2063
2064 static int
2065 instr_table_translate(struct rte_swx_pipeline *p,
2066                       struct action *action,
2067                       char **tokens,
2068                       int n_tokens,
2069                       struct instruction *instr,
2070                       struct instruction_data *data __rte_unused)
2071 {
2072         struct table *t;
2073         struct selector *s;
2074         struct learner *l;
2075
2076         CHECK(!action, EINVAL);
2077         CHECK(n_tokens == 2, EINVAL);
2078
2079         t = table_find(p, tokens[1]);
2080         if (t) {
2081                 instr->type = INSTR_TABLE;
2082                 instr->table.table_id = t->id;
2083                 return 0;
2084         }
2085
2086         s = selector_find(p, tokens[1]);
2087         if (s) {
2088                 instr->type = INSTR_SELECTOR;
2089                 instr->table.table_id = s->id;
2090                 return 0;
2091         }
2092
2093         l = learner_find(p, tokens[1]);
2094         if (l) {
2095                 instr->type = INSTR_LEARNER;
2096                 instr->table.table_id = l->id;
2097                 return 0;
2098         }
2099
2100         CHECK(0, EINVAL);
2101 }
2102
2103 static inline void
2104 instr_table_exec(struct rte_swx_pipeline *p)
2105 {
2106         struct thread *t = &p->threads[p->thread_id];
2107         struct instruction *ip = t->ip;
2108         uint32_t table_id = ip->table.table_id;
2109         struct rte_swx_table_state *ts = &t->table_state[table_id];
2110         struct table_runtime *table = &t->tables[table_id];
2111         struct table_statistics *stats = &p->table_stats[table_id];
2112         uint64_t action_id, n_pkts_hit, n_pkts_action;
2113         uint8_t *action_data;
2114         int done, hit;
2115
2116         /* Table. */
2117         done = table->func(ts->obj,
2118                            table->mailbox,
2119                            table->key,
2120                            &action_id,
2121                            &action_data,
2122                            &hit);
2123         if (!done) {
2124                 /* Thread. */
2125                 TRACE("[Thread %2u] table %u (not finalized)\n",
2126                       p->thread_id,
2127                       table_id);
2128
2129                 thread_yield(p);
2130                 return;
2131         }
2132
2133         action_id = hit ? action_id : ts->default_action_id;
2134         action_data = hit ? action_data : ts->default_action_data;
2135         n_pkts_hit = stats->n_pkts_hit[hit];
2136         n_pkts_action = stats->n_pkts_action[action_id];
2137
2138         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2139               p->thread_id,
2140               table_id,
2141               hit ? "hit" : "miss",
2142               (uint32_t)action_id);
2143
2144         t->action_id = action_id;
2145         t->structs[0] = action_data;
2146         t->hit = hit;
2147         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2148         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2149
2150         /* Thread. */
2151         thread_ip_action_call(p, t, action_id);
2152 }
2153
2154 static inline void
2155 instr_selector_exec(struct rte_swx_pipeline *p)
2156 {
2157         struct thread *t = &p->threads[p->thread_id];
2158         struct instruction *ip = t->ip;
2159         uint32_t selector_id = ip->table.table_id;
2160         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2161         struct selector_runtime *selector = &t->selectors[selector_id];
2162         struct selector_statistics *stats = &p->selector_stats[selector_id];
2163         uint64_t n_pkts = stats->n_pkts;
2164         int done;
2165
2166         /* Table. */
2167         done = rte_swx_table_selector_select(ts->obj,
2168                            selector->mailbox,
2169                            selector->group_id_buffer,
2170                            selector->selector_buffer,
2171                            selector->member_id_buffer);
2172         if (!done) {
2173                 /* Thread. */
2174                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2175                       p->thread_id,
2176                       selector_id);
2177
2178                 thread_yield(p);
2179                 return;
2180         }
2181
2182
2183         TRACE("[Thread %2u] selector %u\n",
2184               p->thread_id,
2185               selector_id);
2186
2187         stats->n_pkts = n_pkts + 1;
2188
2189         /* Thread. */
2190         thread_ip_inc(p);
2191 }
2192
2193 static inline void
2194 instr_learner_exec(struct rte_swx_pipeline *p)
2195 {
2196         struct thread *t = &p->threads[p->thread_id];
2197         struct instruction *ip = t->ip;
2198         uint32_t learner_id = ip->table.table_id;
2199         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2200                 p->n_selectors + learner_id];
2201         struct learner_runtime *l = &t->learners[learner_id];
2202         struct learner_statistics *stats = &p->learner_stats[learner_id];
2203         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2204         uint8_t *action_data;
2205         int done, hit;
2206
2207         /* Table. */
2208         time = rte_get_tsc_cycles();
2209
2210         done = rte_swx_table_learner_lookup(ts->obj,
2211                                             l->mailbox,
2212                                             time,
2213                                             l->key,
2214                                             &action_id,
2215                                             &action_data,
2216                                             &hit);
2217         if (!done) {
2218                 /* Thread. */
2219                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2220                       p->thread_id,
2221                       learner_id);
2222
2223                 thread_yield(p);
2224                 return;
2225         }
2226
2227         action_id = hit ? action_id : ts->default_action_id;
2228         action_data = hit ? action_data : ts->default_action_data;
2229         n_pkts_hit = stats->n_pkts_hit[hit];
2230         n_pkts_action = stats->n_pkts_action[action_id];
2231
2232         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2233               p->thread_id,
2234               learner_id,
2235               hit ? "hit" : "miss",
2236               (uint32_t)action_id);
2237
2238         t->action_id = action_id;
2239         t->structs[0] = action_data;
2240         t->hit = hit;
2241         t->learner_id = learner_id;
2242         t->time = time;
2243         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2244         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2245
2246         /* Thread. */
2247         thread_ip_action_call(p, t, action_id);
2248 }
2249
2250 /*
2251  * learn.
2252  */
2253 static struct action *
2254 action_find(struct rte_swx_pipeline *p, const char *name);
2255
2256 static int
2257 action_has_nbo_args(struct action *a);
2258
2259 static int
2260 instr_learn_translate(struct rte_swx_pipeline *p,
2261                       struct action *action,
2262                       char **tokens,
2263                       int n_tokens,
2264                       struct instruction *instr,
2265                       struct instruction_data *data __rte_unused)
2266 {
2267         struct action *a;
2268
2269         CHECK(action, EINVAL);
2270         CHECK(n_tokens == 2, EINVAL);
2271
2272         a = action_find(p, tokens[1]);
2273         CHECK(a, EINVAL);
2274         CHECK(!action_has_nbo_args(a), EINVAL);
2275
2276         instr->type = INSTR_LEARNER_LEARN;
2277         instr->learn.action_id = a->id;
2278
2279         return 0;
2280 }
2281
2282 static inline void
2283 instr_learn_exec(struct rte_swx_pipeline *p)
2284 {
2285         struct thread *t = &p->threads[p->thread_id];
2286         struct instruction *ip = t->ip;
2287         uint64_t action_id = ip->learn.action_id;
2288         uint32_t learner_id = t->learner_id;
2289         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2290                 p->n_selectors + learner_id];
2291         struct learner_runtime *l = &t->learners[learner_id];
2292         struct learner_statistics *stats = &p->learner_stats[learner_id];
2293         uint32_t status;
2294
2295         /* Table. */
2296         status = rte_swx_table_learner_add(ts->obj,
2297                                            l->mailbox,
2298                                            t->time,
2299                                            action_id,
2300                                            l->action_data[action_id]);
2301
2302         TRACE("[Thread %2u] learner %u learn %s\n",
2303               p->thread_id,
2304               learner_id,
2305               status ? "ok" : "error");
2306
2307         stats->n_pkts_learn[status] += 1;
2308
2309         /* Thread. */
2310         thread_ip_inc(p);
2311 }
2312
2313 /*
2314  * forget.
2315  */
2316 static int
2317 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2318                        struct action *action,
2319                        char **tokens __rte_unused,
2320                        int n_tokens,
2321                        struct instruction *instr,
2322                        struct instruction_data *data __rte_unused)
2323 {
2324         CHECK(action, EINVAL);
2325         CHECK(n_tokens == 1, EINVAL);
2326
2327         instr->type = INSTR_LEARNER_FORGET;
2328
2329         return 0;
2330 }
2331
2332 static inline void
2333 instr_forget_exec(struct rte_swx_pipeline *p)
2334 {
2335         struct thread *t = &p->threads[p->thread_id];
2336         uint32_t learner_id = t->learner_id;
2337         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2338                 p->n_selectors + learner_id];
2339         struct learner_runtime *l = &t->learners[learner_id];
2340         struct learner_statistics *stats = &p->learner_stats[learner_id];
2341
2342         /* Table. */
2343         rte_swx_table_learner_delete(ts->obj, l->mailbox);
2344
2345         TRACE("[Thread %2u] learner %u forget\n",
2346               p->thread_id,
2347               learner_id);
2348
2349         stats->n_pkts_forget += 1;
2350
2351         /* Thread. */
2352         thread_ip_inc(p);
2353 }
2354
2355 /*
2356  * extern.
2357  */
2358 static int
2359 instr_extern_translate(struct rte_swx_pipeline *p,
2360                        struct action *action __rte_unused,
2361                        char **tokens,
2362                        int n_tokens,
2363                        struct instruction *instr,
2364                        struct instruction_data *data __rte_unused)
2365 {
2366         char *token = tokens[1];
2367
2368         CHECK(n_tokens == 2, EINVAL);
2369
2370         if (token[0] == 'e') {
2371                 struct extern_obj *obj;
2372                 struct extern_type_member_func *func;
2373
2374                 func = extern_obj_member_func_parse(p, token, &obj);
2375                 CHECK(func, EINVAL);
2376
2377                 instr->type = INSTR_EXTERN_OBJ;
2378                 instr->ext_obj.ext_obj_id = obj->id;
2379                 instr->ext_obj.func_id = func->id;
2380
2381                 return 0;
2382         }
2383
2384         if (token[0] == 'f') {
2385                 struct extern_func *func;
2386
2387                 func = extern_func_parse(p, token);
2388                 CHECK(func, EINVAL);
2389
2390                 instr->type = INSTR_EXTERN_FUNC;
2391                 instr->ext_func.ext_func_id = func->id;
2392
2393                 return 0;
2394         }
2395
2396         CHECK(0, EINVAL);
2397 }
2398
2399 static inline void
2400 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2401 {
2402         struct thread *t = &p->threads[p->thread_id];
2403         struct instruction *ip = t->ip;
2404         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2405         uint32_t func_id = ip->ext_obj.func_id;
2406         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2407         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2408
2409         TRACE("[Thread %2u] extern obj %u member func %u\n",
2410               p->thread_id,
2411               obj_id,
2412               func_id);
2413
2414         /* Extern object member function execute. */
2415         uint32_t done = func(obj->obj, obj->mailbox);
2416
2417         /* Thread. */
2418         thread_ip_inc_cond(t, done);
2419         thread_yield_cond(p, done ^ 1);
2420 }
2421
2422 static inline void
2423 instr_extern_func_exec(struct rte_swx_pipeline *p)
2424 {
2425         struct thread *t = &p->threads[p->thread_id];
2426         struct instruction *ip = t->ip;
2427         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2428         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2429         rte_swx_extern_func_t func = ext_func->func;
2430
2431         TRACE("[Thread %2u] extern func %u\n",
2432               p->thread_id,
2433               ext_func_id);
2434
2435         /* Extern function execute. */
2436         uint32_t done = func(ext_func->mailbox);
2437
2438         /* Thread. */
2439         thread_ip_inc_cond(t, done);
2440         thread_yield_cond(p, done ^ 1);
2441 }
2442
2443 /*
2444  * mov.
2445  */
2446 static int
2447 instr_mov_translate(struct rte_swx_pipeline *p,
2448                     struct action *action,
2449                     char **tokens,
2450                     int n_tokens,
2451                     struct instruction *instr,
2452                     struct instruction_data *data __rte_unused)
2453 {
2454         char *dst = tokens[1], *src = tokens[2];
2455         struct field *fdst, *fsrc;
2456         uint64_t src_val;
2457         uint32_t dst_struct_id = 0, src_struct_id = 0;
2458
2459         CHECK(n_tokens == 3, EINVAL);
2460
2461         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2462         CHECK(fdst, EINVAL);
2463         CHECK(!fdst->var_size, EINVAL);
2464
2465         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2466         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2467         if (fsrc) {
2468                 CHECK(!fsrc->var_size, EINVAL);
2469
2470                 instr->type = INSTR_MOV;
2471                 if (dst[0] != 'h' && src[0] == 'h')
2472                         instr->type = INSTR_MOV_MH;
2473                 if (dst[0] == 'h' && src[0] != 'h')
2474                         instr->type = INSTR_MOV_HM;
2475                 if (dst[0] == 'h' && src[0] == 'h')
2476                         instr->type = INSTR_MOV_HH;
2477
2478                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2479                 instr->mov.dst.n_bits = fdst->n_bits;
2480                 instr->mov.dst.offset = fdst->offset / 8;
2481                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2482                 instr->mov.src.n_bits = fsrc->n_bits;
2483                 instr->mov.src.offset = fsrc->offset / 8;
2484                 return 0;
2485         }
2486
2487         /* MOV_I. */
2488         src_val = strtoull(src, &src, 0);
2489         CHECK(!src[0], EINVAL);
2490
2491         if (dst[0] == 'h')
2492                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2493
2494         instr->type = INSTR_MOV_I;
2495         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2496         instr->mov.dst.n_bits = fdst->n_bits;
2497         instr->mov.dst.offset = fdst->offset / 8;
2498         instr->mov.src_val = src_val;
2499         return 0;
2500 }
2501
2502 static inline void
2503 instr_mov_exec(struct rte_swx_pipeline *p)
2504 {
2505         struct thread *t = &p->threads[p->thread_id];
2506         struct instruction *ip = t->ip;
2507
2508         TRACE("[Thread %2u] mov\n",
2509               p->thread_id);
2510
2511         MOV(t, ip);
2512
2513         /* Thread. */
2514         thread_ip_inc(p);
2515 }
2516
2517 static inline void
2518 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2519 {
2520         struct thread *t = &p->threads[p->thread_id];
2521         struct instruction *ip = t->ip;
2522
2523         TRACE("[Thread %2u] mov (mh)\n",
2524               p->thread_id);
2525
2526         MOV_MH(t, ip);
2527
2528         /* Thread. */
2529         thread_ip_inc(p);
2530 }
2531
2532 static inline void
2533 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2534 {
2535         struct thread *t = &p->threads[p->thread_id];
2536         struct instruction *ip = t->ip;
2537
2538         TRACE("[Thread %2u] mov (hm)\n",
2539               p->thread_id);
2540
2541         MOV_HM(t, ip);
2542
2543         /* Thread. */
2544         thread_ip_inc(p);
2545 }
2546
2547 static inline void
2548 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2549 {
2550         struct thread *t = &p->threads[p->thread_id];
2551         struct instruction *ip = t->ip;
2552
2553         TRACE("[Thread %2u] mov (hh)\n",
2554               p->thread_id);
2555
2556         MOV_HH(t, ip);
2557
2558         /* Thread. */
2559         thread_ip_inc(p);
2560 }
2561
2562 static inline void
2563 instr_mov_i_exec(struct rte_swx_pipeline *p)
2564 {
2565         struct thread *t = &p->threads[p->thread_id];
2566         struct instruction *ip = t->ip;
2567
2568         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2569               p->thread_id,
2570               ip->mov.src_val);
2571
2572         MOV_I(t, ip);
2573
2574         /* Thread. */
2575         thread_ip_inc(p);
2576 }
2577
2578 /*
2579  * dma.
2580  */
2581 static inline void
2582 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2583
2584 static inline void
2585 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2586 {
2587         struct thread *t = &p->threads[p->thread_id];
2588         struct instruction *ip = t->ip;
2589         uint8_t *action_data = t->structs[0];
2590         uint64_t valid_headers = t->valid_headers;
2591         uint32_t i;
2592
2593         for (i = 0; i < n_dma; i++) {
2594                 uint32_t header_id = ip->dma.dst.header_id[i];
2595                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2596                 uint32_t offset = ip->dma.src.offset[i];
2597                 uint32_t n_bytes = ip->dma.n_bytes[i];
2598
2599                 struct header_runtime *h = &t->headers[header_id];
2600                 uint8_t *h_ptr0 = h->ptr0;
2601                 uint8_t *h_ptr = t->structs[struct_id];
2602
2603                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2604                         h_ptr : h_ptr0;
2605                 void *src = &action_data[offset];
2606
2607                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2608
2609                 /* Headers. */
2610                 memcpy(dst, src, n_bytes);
2611                 t->structs[struct_id] = dst;
2612                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2613         }
2614
2615         t->valid_headers = valid_headers;
2616 }
2617
2618 static inline void
2619 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2620 {
2621         __instr_dma_ht_exec(p, 1);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 static inline void
2628 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2629 {
2630         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2631               p->thread_id);
2632
2633         __instr_dma_ht_exec(p, 2);
2634
2635         /* Thread. */
2636         thread_ip_inc(p);
2637 }
2638
2639 static inline void
2640 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2641 {
2642         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2643               p->thread_id);
2644
2645         __instr_dma_ht_exec(p, 3);
2646
2647         /* Thread. */
2648         thread_ip_inc(p);
2649 }
2650
2651 static inline void
2652 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2653 {
2654         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2655               p->thread_id);
2656
2657         __instr_dma_ht_exec(p, 4);
2658
2659         /* Thread. */
2660         thread_ip_inc(p);
2661 }
2662
2663 static inline void
2664 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2665 {
2666         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2667               p->thread_id);
2668
2669         __instr_dma_ht_exec(p, 5);
2670
2671         /* Thread. */
2672         thread_ip_inc(p);
2673 }
2674
2675 static inline void
2676 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2677 {
2678         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2679               p->thread_id);
2680
2681         __instr_dma_ht_exec(p, 6);
2682
2683         /* Thread. */
2684         thread_ip_inc(p);
2685 }
2686
2687 static inline void
2688 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2689 {
2690         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2691               p->thread_id);
2692
2693         __instr_dma_ht_exec(p, 7);
2694
2695         /* Thread. */
2696         thread_ip_inc(p);
2697 }
2698
2699 static inline void
2700 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2701 {
2702         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2703               p->thread_id);
2704
2705         __instr_dma_ht_exec(p, 8);
2706
2707         /* Thread. */
2708         thread_ip_inc(p);
2709 }
2710
2711 /*
2712  * alu.
2713  */
2714 static int
2715 instr_alu_add_translate(struct rte_swx_pipeline *p,
2716                         struct action *action,
2717                         char **tokens,
2718                         int n_tokens,
2719                         struct instruction *instr,
2720                         struct instruction_data *data __rte_unused)
2721 {
2722         char *dst = tokens[1], *src = tokens[2];
2723         struct field *fdst, *fsrc;
2724         uint64_t src_val;
2725         uint32_t dst_struct_id = 0, src_struct_id = 0;
2726
2727         CHECK(n_tokens == 3, EINVAL);
2728
2729         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2730         CHECK(fdst, EINVAL);
2731         CHECK(!fdst->var_size, EINVAL);
2732
2733         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2734         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2735         if (fsrc) {
2736                 CHECK(!fsrc->var_size, EINVAL);
2737
2738                 instr->type = INSTR_ALU_ADD;
2739                 if (dst[0] == 'h' && src[0] != 'h')
2740                         instr->type = INSTR_ALU_ADD_HM;
2741                 if (dst[0] != 'h' && src[0] == 'h')
2742                         instr->type = INSTR_ALU_ADD_MH;
2743                 if (dst[0] == 'h' && src[0] == 'h')
2744                         instr->type = INSTR_ALU_ADD_HH;
2745
2746                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2747                 instr->alu.dst.n_bits = fdst->n_bits;
2748                 instr->alu.dst.offset = fdst->offset / 8;
2749                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2750                 instr->alu.src.n_bits = fsrc->n_bits;
2751                 instr->alu.src.offset = fsrc->offset / 8;
2752                 return 0;
2753         }
2754
2755         /* ADD_MI, ADD_HI. */
2756         src_val = strtoull(src, &src, 0);
2757         CHECK(!src[0], EINVAL);
2758
2759         instr->type = INSTR_ALU_ADD_MI;
2760         if (dst[0] == 'h')
2761                 instr->type = INSTR_ALU_ADD_HI;
2762
2763         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2764         instr->alu.dst.n_bits = fdst->n_bits;
2765         instr->alu.dst.offset = fdst->offset / 8;
2766         instr->alu.src_val = src_val;
2767         return 0;
2768 }
2769
2770 static int
2771 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2772                         struct action *action,
2773                         char **tokens,
2774                         int n_tokens,
2775                         struct instruction *instr,
2776                         struct instruction_data *data __rte_unused)
2777 {
2778         char *dst = tokens[1], *src = tokens[2];
2779         struct field *fdst, *fsrc;
2780         uint64_t src_val;
2781         uint32_t dst_struct_id = 0, src_struct_id = 0;
2782
2783         CHECK(n_tokens == 3, EINVAL);
2784
2785         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2786         CHECK(fdst, EINVAL);
2787         CHECK(!fdst->var_size, EINVAL);
2788
2789         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2790         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2791         if (fsrc) {
2792                 CHECK(!fsrc->var_size, EINVAL);
2793
2794                 instr->type = INSTR_ALU_SUB;
2795                 if (dst[0] == 'h' && src[0] != 'h')
2796                         instr->type = INSTR_ALU_SUB_HM;
2797                 if (dst[0] != 'h' && src[0] == 'h')
2798                         instr->type = INSTR_ALU_SUB_MH;
2799                 if (dst[0] == 'h' && src[0] == 'h')
2800                         instr->type = INSTR_ALU_SUB_HH;
2801
2802                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2803                 instr->alu.dst.n_bits = fdst->n_bits;
2804                 instr->alu.dst.offset = fdst->offset / 8;
2805                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2806                 instr->alu.src.n_bits = fsrc->n_bits;
2807                 instr->alu.src.offset = fsrc->offset / 8;
2808                 return 0;
2809         }
2810
2811         /* SUB_MI, SUB_HI. */
2812         src_val = strtoull(src, &src, 0);
2813         CHECK(!src[0], EINVAL);
2814
2815         instr->type = INSTR_ALU_SUB_MI;
2816         if (dst[0] == 'h')
2817                 instr->type = INSTR_ALU_SUB_HI;
2818
2819         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2820         instr->alu.dst.n_bits = fdst->n_bits;
2821         instr->alu.dst.offset = fdst->offset / 8;
2822         instr->alu.src_val = src_val;
2823         return 0;
2824 }
2825
2826 static int
2827 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2828                           struct action *action __rte_unused,
2829                           char **tokens,
2830                           int n_tokens,
2831                           struct instruction *instr,
2832                           struct instruction_data *data __rte_unused)
2833 {
2834         char *dst = tokens[1], *src = tokens[2];
2835         struct header *hdst, *hsrc;
2836         struct field *fdst, *fsrc;
2837
2838         CHECK(n_tokens == 3, EINVAL);
2839
2840         fdst = header_field_parse(p, dst, &hdst);
2841         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2842         CHECK(!fdst->var_size, EINVAL);
2843
2844         /* CKADD_FIELD. */
2845         fsrc = header_field_parse(p, src, &hsrc);
2846         if (fsrc) {
2847                 CHECK(!fsrc->var_size, EINVAL);
2848
2849                 instr->type = INSTR_ALU_CKADD_FIELD;
2850                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2851                 instr->alu.dst.n_bits = fdst->n_bits;
2852                 instr->alu.dst.offset = fdst->offset / 8;
2853                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2854                 instr->alu.src.n_bits = fsrc->n_bits;
2855                 instr->alu.src.offset = fsrc->offset / 8;
2856                 return 0;
2857         }
2858
2859         /* CKADD_STRUCT, CKADD_STRUCT20. */
2860         hsrc = header_parse(p, src);
2861         CHECK(hsrc, EINVAL);
2862         CHECK(!hsrc->st->var_size, EINVAL);
2863
2864         instr->type = INSTR_ALU_CKADD_STRUCT;
2865         if ((hsrc->st->n_bits / 8) == 20)
2866                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2867
2868         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2869         instr->alu.dst.n_bits = fdst->n_bits;
2870         instr->alu.dst.offset = fdst->offset / 8;
2871         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2872         instr->alu.src.n_bits = hsrc->st->n_bits;
2873         instr->alu.src.offset = 0; /* Unused. */
2874         return 0;
2875 }
2876
2877 static int
2878 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2879                           struct action *action __rte_unused,
2880                           char **tokens,
2881                           int n_tokens,
2882                           struct instruction *instr,
2883                           struct instruction_data *data __rte_unused)
2884 {
2885         char *dst = tokens[1], *src = tokens[2];
2886         struct header *hdst, *hsrc;
2887         struct field *fdst, *fsrc;
2888
2889         CHECK(n_tokens == 3, EINVAL);
2890
2891         fdst = header_field_parse(p, dst, &hdst);
2892         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2893         CHECK(!fdst->var_size, EINVAL);
2894
2895         fsrc = header_field_parse(p, src, &hsrc);
2896         CHECK(fsrc, EINVAL);
2897         CHECK(!fsrc->var_size, EINVAL);
2898
2899         instr->type = INSTR_ALU_CKSUB_FIELD;
2900         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2901         instr->alu.dst.n_bits = fdst->n_bits;
2902         instr->alu.dst.offset = fdst->offset / 8;
2903         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2904         instr->alu.src.n_bits = fsrc->n_bits;
2905         instr->alu.src.offset = fsrc->offset / 8;
2906         return 0;
2907 }
2908
2909 static int
2910 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2911                         struct action *action,
2912                         char **tokens,
2913                         int n_tokens,
2914                         struct instruction *instr,
2915                         struct instruction_data *data __rte_unused)
2916 {
2917         char *dst = tokens[1], *src = tokens[2];
2918         struct field *fdst, *fsrc;
2919         uint64_t src_val;
2920         uint32_t dst_struct_id = 0, src_struct_id = 0;
2921
2922         CHECK(n_tokens == 3, EINVAL);
2923
2924         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2925         CHECK(fdst, EINVAL);
2926         CHECK(!fdst->var_size, EINVAL);
2927
2928         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2929         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2930         if (fsrc) {
2931                 CHECK(!fsrc->var_size, EINVAL);
2932
2933                 instr->type = INSTR_ALU_SHL;
2934                 if (dst[0] == 'h' && src[0] != 'h')
2935                         instr->type = INSTR_ALU_SHL_HM;
2936                 if (dst[0] != 'h' && src[0] == 'h')
2937                         instr->type = INSTR_ALU_SHL_MH;
2938                 if (dst[0] == 'h' && src[0] == 'h')
2939                         instr->type = INSTR_ALU_SHL_HH;
2940
2941                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2942                 instr->alu.dst.n_bits = fdst->n_bits;
2943                 instr->alu.dst.offset = fdst->offset / 8;
2944                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2945                 instr->alu.src.n_bits = fsrc->n_bits;
2946                 instr->alu.src.offset = fsrc->offset / 8;
2947                 return 0;
2948         }
2949
2950         /* SHL_MI, SHL_HI. */
2951         src_val = strtoull(src, &src, 0);
2952         CHECK(!src[0], EINVAL);
2953
2954         instr->type = INSTR_ALU_SHL_MI;
2955         if (dst[0] == 'h')
2956                 instr->type = INSTR_ALU_SHL_HI;
2957
2958         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2959         instr->alu.dst.n_bits = fdst->n_bits;
2960         instr->alu.dst.offset = fdst->offset / 8;
2961         instr->alu.src_val = src_val;
2962         return 0;
2963 }
2964
2965 static int
2966 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2967                         struct action *action,
2968                         char **tokens,
2969                         int n_tokens,
2970                         struct instruction *instr,
2971                         struct instruction_data *data __rte_unused)
2972 {
2973         char *dst = tokens[1], *src = tokens[2];
2974         struct field *fdst, *fsrc;
2975         uint64_t src_val;
2976         uint32_t dst_struct_id = 0, src_struct_id = 0;
2977
2978         CHECK(n_tokens == 3, EINVAL);
2979
2980         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2981         CHECK(fdst, EINVAL);
2982         CHECK(!fdst->var_size, EINVAL);
2983
2984         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2985         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2986         if (fsrc) {
2987                 CHECK(!fsrc->var_size, EINVAL);
2988
2989                 instr->type = INSTR_ALU_SHR;
2990                 if (dst[0] == 'h' && src[0] != 'h')
2991                         instr->type = INSTR_ALU_SHR_HM;
2992                 if (dst[0] != 'h' && src[0] == 'h')
2993                         instr->type = INSTR_ALU_SHR_MH;
2994                 if (dst[0] == 'h' && src[0] == 'h')
2995                         instr->type = INSTR_ALU_SHR_HH;
2996
2997                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2998                 instr->alu.dst.n_bits = fdst->n_bits;
2999                 instr->alu.dst.offset = fdst->offset / 8;
3000                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3001                 instr->alu.src.n_bits = fsrc->n_bits;
3002                 instr->alu.src.offset = fsrc->offset / 8;
3003                 return 0;
3004         }
3005
3006         /* SHR_MI, SHR_HI. */
3007         src_val = strtoull(src, &src, 0);
3008         CHECK(!src[0], EINVAL);
3009
3010         instr->type = INSTR_ALU_SHR_MI;
3011         if (dst[0] == 'h')
3012                 instr->type = INSTR_ALU_SHR_HI;
3013
3014         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3015         instr->alu.dst.n_bits = fdst->n_bits;
3016         instr->alu.dst.offset = fdst->offset / 8;
3017         instr->alu.src_val = src_val;
3018         return 0;
3019 }
3020
3021 static int
3022 instr_alu_and_translate(struct rte_swx_pipeline *p,
3023                         struct action *action,
3024                         char **tokens,
3025                         int n_tokens,
3026                         struct instruction *instr,
3027                         struct instruction_data *data __rte_unused)
3028 {
3029         char *dst = tokens[1], *src = tokens[2];
3030         struct field *fdst, *fsrc;
3031         uint64_t src_val;
3032         uint32_t dst_struct_id = 0, src_struct_id = 0;
3033
3034         CHECK(n_tokens == 3, EINVAL);
3035
3036         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3037         CHECK(fdst, EINVAL);
3038         CHECK(!fdst->var_size, EINVAL);
3039
3040         /* AND, AND_MH, AND_HM, AND_HH. */
3041         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3042         if (fsrc) {
3043                 CHECK(!fsrc->var_size, EINVAL);
3044
3045                 instr->type = INSTR_ALU_AND;
3046                 if (dst[0] != 'h' && src[0] == 'h')
3047                         instr->type = INSTR_ALU_AND_MH;
3048                 if (dst[0] == 'h' && src[0] != 'h')
3049                         instr->type = INSTR_ALU_AND_HM;
3050                 if (dst[0] == 'h' && src[0] == 'h')
3051                         instr->type = INSTR_ALU_AND_HH;
3052
3053                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3054                 instr->alu.dst.n_bits = fdst->n_bits;
3055                 instr->alu.dst.offset = fdst->offset / 8;
3056                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3057                 instr->alu.src.n_bits = fsrc->n_bits;
3058                 instr->alu.src.offset = fsrc->offset / 8;
3059                 return 0;
3060         }
3061
3062         /* AND_I. */
3063         src_val = strtoull(src, &src, 0);
3064         CHECK(!src[0], EINVAL);
3065
3066         if (dst[0] == 'h')
3067                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3068
3069         instr->type = INSTR_ALU_AND_I;
3070         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3071         instr->alu.dst.n_bits = fdst->n_bits;
3072         instr->alu.dst.offset = fdst->offset / 8;
3073         instr->alu.src_val = src_val;
3074         return 0;
3075 }
3076
3077 static int
3078 instr_alu_or_translate(struct rte_swx_pipeline *p,
3079                        struct action *action,
3080                        char **tokens,
3081                        int n_tokens,
3082                        struct instruction *instr,
3083                        struct instruction_data *data __rte_unused)
3084 {
3085         char *dst = tokens[1], *src = tokens[2];
3086         struct field *fdst, *fsrc;
3087         uint64_t src_val;
3088         uint32_t dst_struct_id = 0, src_struct_id = 0;
3089
3090         CHECK(n_tokens == 3, EINVAL);
3091
3092         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3093         CHECK(fdst, EINVAL);
3094         CHECK(!fdst->var_size, EINVAL);
3095
3096         /* OR, OR_MH, OR_HM, OR_HH. */
3097         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3098         if (fsrc) {
3099                 CHECK(!fsrc->var_size, EINVAL);
3100
3101                 instr->type = INSTR_ALU_OR;
3102                 if (dst[0] != 'h' && src[0] == 'h')
3103                         instr->type = INSTR_ALU_OR_MH;
3104                 if (dst[0] == 'h' && src[0] != 'h')
3105                         instr->type = INSTR_ALU_OR_HM;
3106                 if (dst[0] == 'h' && src[0] == 'h')
3107                         instr->type = INSTR_ALU_OR_HH;
3108
3109                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3110                 instr->alu.dst.n_bits = fdst->n_bits;
3111                 instr->alu.dst.offset = fdst->offset / 8;
3112                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3113                 instr->alu.src.n_bits = fsrc->n_bits;
3114                 instr->alu.src.offset = fsrc->offset / 8;
3115                 return 0;
3116         }
3117
3118         /* OR_I. */
3119         src_val = strtoull(src, &src, 0);
3120         CHECK(!src[0], EINVAL);
3121
3122         if (dst[0] == 'h')
3123                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3124
3125         instr->type = INSTR_ALU_OR_I;
3126         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3127         instr->alu.dst.n_bits = fdst->n_bits;
3128         instr->alu.dst.offset = fdst->offset / 8;
3129         instr->alu.src_val = src_val;
3130         return 0;
3131 }
3132
3133 static int
3134 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3135                         struct action *action,
3136                         char **tokens,
3137                         int n_tokens,
3138                         struct instruction *instr,
3139                         struct instruction_data *data __rte_unused)
3140 {
3141         char *dst = tokens[1], *src = tokens[2];
3142         struct field *fdst, *fsrc;
3143         uint64_t src_val;
3144         uint32_t dst_struct_id = 0, src_struct_id = 0;
3145
3146         CHECK(n_tokens == 3, EINVAL);
3147
3148         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3149         CHECK(fdst, EINVAL);
3150         CHECK(!fdst->var_size, EINVAL);
3151
3152         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3153         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3154         if (fsrc) {
3155                 CHECK(!fsrc->var_size, EINVAL);
3156
3157                 instr->type = INSTR_ALU_XOR;
3158                 if (dst[0] != 'h' && src[0] == 'h')
3159                         instr->type = INSTR_ALU_XOR_MH;
3160                 if (dst[0] == 'h' && src[0] != 'h')
3161                         instr->type = INSTR_ALU_XOR_HM;
3162                 if (dst[0] == 'h' && src[0] == 'h')
3163                         instr->type = INSTR_ALU_XOR_HH;
3164
3165                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3166                 instr->alu.dst.n_bits = fdst->n_bits;
3167                 instr->alu.dst.offset = fdst->offset / 8;
3168                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3169                 instr->alu.src.n_bits = fsrc->n_bits;
3170                 instr->alu.src.offset = fsrc->offset / 8;
3171                 return 0;
3172         }
3173
3174         /* XOR_I. */
3175         src_val = strtoull(src, &src, 0);
3176         CHECK(!src[0], EINVAL);
3177
3178         if (dst[0] == 'h')
3179                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3180
3181         instr->type = INSTR_ALU_XOR_I;
3182         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3183         instr->alu.dst.n_bits = fdst->n_bits;
3184         instr->alu.dst.offset = fdst->offset / 8;
3185         instr->alu.src_val = src_val;
3186         return 0;
3187 }
3188
3189 static inline void
3190 instr_alu_add_exec(struct rte_swx_pipeline *p)
3191 {
3192         struct thread *t = &p->threads[p->thread_id];
3193         struct instruction *ip = t->ip;
3194
3195         TRACE("[Thread %2u] add\n", p->thread_id);
3196
3197         /* Structs. */
3198         ALU(t, ip, +);
3199
3200         /* Thread. */
3201         thread_ip_inc(p);
3202 }
3203
3204 static inline void
3205 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3206 {
3207         struct thread *t = &p->threads[p->thread_id];
3208         struct instruction *ip = t->ip;
3209
3210         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3211
3212         /* Structs. */
3213         ALU_MH(t, ip, +);
3214
3215         /* Thread. */
3216         thread_ip_inc(p);
3217 }
3218
3219 static inline void
3220 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3221 {
3222         struct thread *t = &p->threads[p->thread_id];
3223         struct instruction *ip = t->ip;
3224
3225         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3226
3227         /* Structs. */
3228         ALU_HM(t, ip, +);
3229
3230         /* Thread. */
3231         thread_ip_inc(p);
3232 }
3233
3234 static inline void
3235 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3236 {
3237         struct thread *t = &p->threads[p->thread_id];
3238         struct instruction *ip = t->ip;
3239
3240         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3241
3242         /* Structs. */
3243         ALU_HH(t, ip, +);
3244
3245         /* Thread. */
3246         thread_ip_inc(p);
3247 }
3248
3249 static inline void
3250 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3251 {
3252         struct thread *t = &p->threads[p->thread_id];
3253         struct instruction *ip = t->ip;
3254
3255         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3256
3257         /* Structs. */
3258         ALU_MI(t, ip, +);
3259
3260         /* Thread. */
3261         thread_ip_inc(p);
3262 }
3263
3264 static inline void
3265 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3266 {
3267         struct thread *t = &p->threads[p->thread_id];
3268         struct instruction *ip = t->ip;
3269
3270         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3271
3272         /* Structs. */
3273         ALU_HI(t, ip, +);
3274
3275         /* Thread. */
3276         thread_ip_inc(p);
3277 }
3278
3279 static inline void
3280 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3281 {
3282         struct thread *t = &p->threads[p->thread_id];
3283         struct instruction *ip = t->ip;
3284
3285         TRACE("[Thread %2u] sub\n", p->thread_id);
3286
3287         /* Structs. */
3288         ALU(t, ip, -);
3289
3290         /* Thread. */
3291         thread_ip_inc(p);
3292 }
3293
3294 static inline void
3295 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3296 {
3297         struct thread *t = &p->threads[p->thread_id];
3298         struct instruction *ip = t->ip;
3299
3300         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3301
3302         /* Structs. */
3303         ALU_MH(t, ip, -);
3304
3305         /* Thread. */
3306         thread_ip_inc(p);
3307 }
3308
3309 static inline void
3310 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3311 {
3312         struct thread *t = &p->threads[p->thread_id];
3313         struct instruction *ip = t->ip;
3314
3315         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3316
3317         /* Structs. */
3318         ALU_HM(t, ip, -);
3319
3320         /* Thread. */
3321         thread_ip_inc(p);
3322 }
3323
3324 static inline void
3325 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3326 {
3327         struct thread *t = &p->threads[p->thread_id];
3328         struct instruction *ip = t->ip;
3329
3330         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3331
3332         /* Structs. */
3333         ALU_HH(t, ip, -);
3334
3335         /* Thread. */
3336         thread_ip_inc(p);
3337 }
3338
3339 static inline void
3340 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3341 {
3342         struct thread *t = &p->threads[p->thread_id];
3343         struct instruction *ip = t->ip;
3344
3345         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3346
3347         /* Structs. */
3348         ALU_MI(t, ip, -);
3349
3350         /* Thread. */
3351         thread_ip_inc(p);
3352 }
3353
3354 static inline void
3355 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3356 {
3357         struct thread *t = &p->threads[p->thread_id];
3358         struct instruction *ip = t->ip;
3359
3360         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3361
3362         /* Structs. */
3363         ALU_HI(t, ip, -);
3364
3365         /* Thread. */
3366         thread_ip_inc(p);
3367 }
3368
3369 static inline void
3370 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3371 {
3372         struct thread *t = &p->threads[p->thread_id];
3373         struct instruction *ip = t->ip;
3374
3375         TRACE("[Thread %2u] shl\n", p->thread_id);
3376
3377         /* Structs. */
3378         ALU(t, ip, <<);
3379
3380         /* Thread. */
3381         thread_ip_inc(p);
3382 }
3383
3384 static inline void
3385 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3386 {
3387         struct thread *t = &p->threads[p->thread_id];
3388         struct instruction *ip = t->ip;
3389
3390         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3391
3392         /* Structs. */
3393         ALU_MH(t, ip, <<);
3394
3395         /* Thread. */
3396         thread_ip_inc(p);
3397 }
3398
3399 static inline void
3400 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3401 {
3402         struct thread *t = &p->threads[p->thread_id];
3403         struct instruction *ip = t->ip;
3404
3405         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3406
3407         /* Structs. */
3408         ALU_HM(t, ip, <<);
3409
3410         /* Thread. */
3411         thread_ip_inc(p);
3412 }
3413
3414 static inline void
3415 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3416 {
3417         struct thread *t = &p->threads[p->thread_id];
3418         struct instruction *ip = t->ip;
3419
3420         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3421
3422         /* Structs. */
3423         ALU_HH(t, ip, <<);
3424
3425         /* Thread. */
3426         thread_ip_inc(p);
3427 }
3428
3429 static inline void
3430 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3431 {
3432         struct thread *t = &p->threads[p->thread_id];
3433         struct instruction *ip = t->ip;
3434
3435         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3436
3437         /* Structs. */
3438         ALU_MI(t, ip, <<);
3439
3440         /* Thread. */
3441         thread_ip_inc(p);
3442 }
3443
3444 static inline void
3445 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3446 {
3447         struct thread *t = &p->threads[p->thread_id];
3448         struct instruction *ip = t->ip;
3449
3450         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3451
3452         /* Structs. */
3453         ALU_HI(t, ip, <<);
3454
3455         /* Thread. */
3456         thread_ip_inc(p);
3457 }
3458
3459 static inline void
3460 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3461 {
3462         struct thread *t = &p->threads[p->thread_id];
3463         struct instruction *ip = t->ip;
3464
3465         TRACE("[Thread %2u] shr\n", p->thread_id);
3466
3467         /* Structs. */
3468         ALU(t, ip, >>);
3469
3470         /* Thread. */
3471         thread_ip_inc(p);
3472 }
3473
3474 static inline void
3475 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3476 {
3477         struct thread *t = &p->threads[p->thread_id];
3478         struct instruction *ip = t->ip;
3479
3480         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3481
3482         /* Structs. */
3483         ALU_MH(t, ip, >>);
3484
3485         /* Thread. */
3486         thread_ip_inc(p);
3487 }
3488
3489 static inline void
3490 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3491 {
3492         struct thread *t = &p->threads[p->thread_id];
3493         struct instruction *ip = t->ip;
3494
3495         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3496
3497         /* Structs. */
3498         ALU_HM(t, ip, >>);
3499
3500         /* Thread. */
3501         thread_ip_inc(p);
3502 }
3503
3504 static inline void
3505 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3506 {
3507         struct thread *t = &p->threads[p->thread_id];
3508         struct instruction *ip = t->ip;
3509
3510         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3511
3512         /* Structs. */
3513         ALU_HH(t, ip, >>);
3514
3515         /* Thread. */
3516         thread_ip_inc(p);
3517 }
3518
3519 static inline void
3520 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3521 {
3522         struct thread *t = &p->threads[p->thread_id];
3523         struct instruction *ip = t->ip;
3524
3525         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3526
3527         /* Structs. */
3528         ALU_MI(t, ip, >>);
3529
3530         /* Thread. */
3531         thread_ip_inc(p);
3532 }
3533
3534 static inline void
3535 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3536 {
3537         struct thread *t = &p->threads[p->thread_id];
3538         struct instruction *ip = t->ip;
3539
3540         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3541
3542         /* Structs. */
3543         ALU_HI(t, ip, >>);
3544
3545         /* Thread. */
3546         thread_ip_inc(p);
3547 }
3548
3549 static inline void
3550 instr_alu_and_exec(struct rte_swx_pipeline *p)
3551 {
3552         struct thread *t = &p->threads[p->thread_id];
3553         struct instruction *ip = t->ip;
3554
3555         TRACE("[Thread %2u] and\n", p->thread_id);
3556
3557         /* Structs. */
3558         ALU(t, ip, &);
3559
3560         /* Thread. */
3561         thread_ip_inc(p);
3562 }
3563
3564 static inline void
3565 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3566 {
3567         struct thread *t = &p->threads[p->thread_id];
3568         struct instruction *ip = t->ip;
3569
3570         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3571
3572         /* Structs. */
3573         ALU_MH(t, ip, &);
3574
3575         /* Thread. */
3576         thread_ip_inc(p);
3577 }
3578
3579 static inline void
3580 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3581 {
3582         struct thread *t = &p->threads[p->thread_id];
3583         struct instruction *ip = t->ip;
3584
3585         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3586
3587         /* Structs. */
3588         ALU_HM_FAST(t, ip, &);
3589
3590         /* Thread. */
3591         thread_ip_inc(p);
3592 }
3593
3594 static inline void
3595 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3596 {
3597         struct thread *t = &p->threads[p->thread_id];
3598         struct instruction *ip = t->ip;
3599
3600         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3601
3602         /* Structs. */
3603         ALU_HH_FAST(t, ip, &);
3604
3605         /* Thread. */
3606         thread_ip_inc(p);
3607 }
3608
3609 static inline void
3610 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3611 {
3612         struct thread *t = &p->threads[p->thread_id];
3613         struct instruction *ip = t->ip;
3614
3615         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3616
3617         /* Structs. */
3618         ALU_I(t, ip, &);
3619
3620         /* Thread. */
3621         thread_ip_inc(p);
3622 }
3623
3624 static inline void
3625 instr_alu_or_exec(struct rte_swx_pipeline *p)
3626 {
3627         struct thread *t = &p->threads[p->thread_id];
3628         struct instruction *ip = t->ip;
3629
3630         TRACE("[Thread %2u] or\n", p->thread_id);
3631
3632         /* Structs. */
3633         ALU(t, ip, |);
3634
3635         /* Thread. */
3636         thread_ip_inc(p);
3637 }
3638
3639 static inline void
3640 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3641 {
3642         struct thread *t = &p->threads[p->thread_id];
3643         struct instruction *ip = t->ip;
3644
3645         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3646
3647         /* Structs. */
3648         ALU_MH(t, ip, |);
3649
3650         /* Thread. */
3651         thread_ip_inc(p);
3652 }
3653
3654 static inline void
3655 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3656 {
3657         struct thread *t = &p->threads[p->thread_id];
3658         struct instruction *ip = t->ip;
3659
3660         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3661
3662         /* Structs. */
3663         ALU_HM_FAST(t, ip, |);
3664
3665         /* Thread. */
3666         thread_ip_inc(p);
3667 }
3668
3669 static inline void
3670 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3671 {
3672         struct thread *t = &p->threads[p->thread_id];
3673         struct instruction *ip = t->ip;
3674
3675         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3676
3677         /* Structs. */
3678         ALU_HH_FAST(t, ip, |);
3679
3680         /* Thread. */
3681         thread_ip_inc(p);
3682 }
3683
3684 static inline void
3685 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3686 {
3687         struct thread *t = &p->threads[p->thread_id];
3688         struct instruction *ip = t->ip;
3689
3690         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3691
3692         /* Structs. */
3693         ALU_I(t, ip, |);
3694
3695         /* Thread. */
3696         thread_ip_inc(p);
3697 }
3698
3699 static inline void
3700 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3701 {
3702         struct thread *t = &p->threads[p->thread_id];
3703         struct instruction *ip = t->ip;
3704
3705         TRACE("[Thread %2u] xor\n", p->thread_id);
3706
3707         /* Structs. */
3708         ALU(t, ip, ^);
3709
3710         /* Thread. */
3711         thread_ip_inc(p);
3712 }
3713
3714 static inline void
3715 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3716 {
3717         struct thread *t = &p->threads[p->thread_id];
3718         struct instruction *ip = t->ip;
3719
3720         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3721
3722         /* Structs. */
3723         ALU_MH(t, ip, ^);
3724
3725         /* Thread. */
3726         thread_ip_inc(p);
3727 }
3728
3729 static inline void
3730 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3731 {
3732         struct thread *t = &p->threads[p->thread_id];
3733         struct instruction *ip = t->ip;
3734
3735         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3736
3737         /* Structs. */
3738         ALU_HM_FAST(t, ip, ^);
3739
3740         /* Thread. */
3741         thread_ip_inc(p);
3742 }
3743
3744 static inline void
3745 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3746 {
3747         struct thread *t = &p->threads[p->thread_id];
3748         struct instruction *ip = t->ip;
3749
3750         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3751
3752         /* Structs. */
3753         ALU_HH_FAST(t, ip, ^);
3754
3755         /* Thread. */
3756         thread_ip_inc(p);
3757 }
3758
3759 static inline void
3760 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3761 {
3762         struct thread *t = &p->threads[p->thread_id];
3763         struct instruction *ip = t->ip;
3764
3765         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3766
3767         /* Structs. */
3768         ALU_I(t, ip, ^);
3769
3770         /* Thread. */
3771         thread_ip_inc(p);
3772 }
3773
3774 static inline void
3775 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3776 {
3777         struct thread *t = &p->threads[p->thread_id];
3778         struct instruction *ip = t->ip;
3779         uint8_t *dst_struct, *src_struct;
3780         uint16_t *dst16_ptr, dst;
3781         uint64_t *src64_ptr, src64, src64_mask, src;
3782         uint64_t r;
3783
3784         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3785
3786         /* Structs. */
3787         dst_struct = t->structs[ip->alu.dst.struct_id];
3788         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3789         dst = *dst16_ptr;
3790
3791         src_struct = t->structs[ip->alu.src.struct_id];
3792         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3793         src64 = *src64_ptr;
3794         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3795         src = src64 & src64_mask;
3796
3797         r = dst;
3798         r = ~r & 0xFFFF;
3799
3800         /* The first input (r) is a 16-bit number. The second and the third
3801          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3802          * three numbers (output r) is a 34-bit number.
3803          */
3804         r += (src >> 32) + (src & 0xFFFFFFFF);
3805
3806         /* The first input is a 16-bit number. The second input is an 18-bit
3807          * number. In the worst case scenario, the sum of the two numbers is a
3808          * 19-bit number.
3809          */
3810         r = (r & 0xFFFF) + (r >> 16);
3811
3812         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3813          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3814          */
3815         r = (r & 0xFFFF) + (r >> 16);
3816
3817         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3818          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3819          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3820          * therefore the output r is always a 16-bit number.
3821          */
3822         r = (r & 0xFFFF) + (r >> 16);
3823
3824         r = ~r & 0xFFFF;
3825         r = r ? r : 0xFFFF;
3826
3827         *dst16_ptr = (uint16_t)r;
3828
3829         /* Thread. */
3830         thread_ip_inc(p);
3831 }
3832
3833 static inline void
3834 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3835 {
3836         struct thread *t = &p->threads[p->thread_id];
3837         struct instruction *ip = t->ip;
3838         uint8_t *dst_struct, *src_struct;
3839         uint16_t *dst16_ptr, dst;
3840         uint64_t *src64_ptr, src64, src64_mask, src;
3841         uint64_t r;
3842
3843         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3844
3845         /* Structs. */
3846         dst_struct = t->structs[ip->alu.dst.struct_id];
3847         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3848         dst = *dst16_ptr;
3849
3850         src_struct = t->structs[ip->alu.src.struct_id];
3851         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3852         src64 = *src64_ptr;
3853         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3854         src = src64 & src64_mask;
3855
3856         r = dst;
3857         r = ~r & 0xFFFF;
3858
3859         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3860          * the following sequence of operations in 2's complement arithmetic:
3861          *    a '- b = (a - b) % 0xFFFF.
3862          *
3863          * In order to prevent an underflow for the below subtraction, in which
3864          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3865          * minuend), we first add a multiple of the 0xFFFF modulus to the
3866          * minuend. The number we add to the minuend needs to be a 34-bit number
3867          * or higher, so for readability reasons we picked the 36-bit multiple.
3868          * We are effectively turning the 16-bit minuend into a 36-bit number:
3869          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3870          */
3871         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3872
3873         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3874          * result (the output r) is a 36-bit number.
3875          */
3876         r -= (src >> 32) + (src & 0xFFFFFFFF);
3877
3878         /* The first input is a 16-bit number. The second input is a 20-bit
3879          * number. Their sum is a 21-bit number.
3880          */
3881         r = (r & 0xFFFF) + (r >> 16);
3882
3883         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3884          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3885          */
3886         r = (r & 0xFFFF) + (r >> 16);
3887
3888         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3889          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3890          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3891          * generated, therefore the output r is always a 16-bit number.
3892          */
3893         r = (r & 0xFFFF) + (r >> 16);
3894
3895         r = ~r & 0xFFFF;
3896         r = r ? r : 0xFFFF;
3897
3898         *dst16_ptr = (uint16_t)r;
3899
3900         /* Thread. */
3901         thread_ip_inc(p);
3902 }
3903
3904 static inline void
3905 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3906 {
3907         struct thread *t = &p->threads[p->thread_id];
3908         struct instruction *ip = t->ip;
3909         uint8_t *dst_struct, *src_struct;
3910         uint16_t *dst16_ptr;
3911         uint32_t *src32_ptr;
3912         uint64_t r0, r1;
3913
3914         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3915
3916         /* Structs. */
3917         dst_struct = t->structs[ip->alu.dst.struct_id];
3918         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3919
3920         src_struct = t->structs[ip->alu.src.struct_id];
3921         src32_ptr = (uint32_t *)&src_struct[0];
3922
3923         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3924         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3925         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3926         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3927         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3928
3929         /* The first input is a 16-bit number. The second input is a 19-bit
3930          * number. Their sum is a 20-bit number.
3931          */
3932         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3933
3934         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3935          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3936          */
3937         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3938
3939         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3940          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3941          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3942          * generated, therefore the output r is always a 16-bit number.
3943          */
3944         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3945
3946         r0 = ~r0 & 0xFFFF;
3947         r0 = r0 ? r0 : 0xFFFF;
3948
3949         *dst16_ptr = (uint16_t)r0;
3950
3951         /* Thread. */
3952         thread_ip_inc(p);
3953 }
3954
3955 static inline void
3956 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3957 {
3958         struct thread *t = &p->threads[p->thread_id];
3959         struct instruction *ip = t->ip;
3960         uint8_t *dst_struct, *src_struct;
3961         uint16_t *dst16_ptr;
3962         uint32_t *src32_ptr;
3963         uint64_t r = 0;
3964         uint32_t i;
3965
3966         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3967
3968         /* Structs. */
3969         dst_struct = t->structs[ip->alu.dst.struct_id];
3970         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3971
3972         src_struct = t->structs[ip->alu.src.struct_id];
3973         src32_ptr = (uint32_t *)&src_struct[0];
3974
3975         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3976          * Therefore, in the worst case scenario, a 35-bit number is added to a
3977          * 16-bit number (the input r), so the output r is 36-bit number.
3978          */
3979         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3980                 r += *src32_ptr;
3981
3982         /* The first input is a 16-bit number. The second input is a 20-bit
3983          * number. Their sum is a 21-bit number.
3984          */
3985         r = (r & 0xFFFF) + (r >> 16);
3986
3987         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3988          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3989          */
3990         r = (r & 0xFFFF) + (r >> 16);
3991
3992         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3993          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3994          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3995          * generated, therefore the output r is always a 16-bit number.
3996          */
3997         r = (r & 0xFFFF) + (r >> 16);
3998
3999         r = ~r & 0xFFFF;
4000         r = r ? r : 0xFFFF;
4001
4002         *dst16_ptr = (uint16_t)r;
4003
4004         /* Thread. */
4005         thread_ip_inc(p);
4006 }
4007
4008 /*
4009  * Register array.
4010  */
4011 static struct regarray *
4012 regarray_find(struct rte_swx_pipeline *p, const char *name);
4013
4014 static int
4015 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4016                       struct action *action,
4017                       char **tokens,
4018                       int n_tokens,
4019                       struct instruction *instr,
4020                       struct instruction_data *data __rte_unused)
4021 {
4022         char *regarray = tokens[1], *idx = tokens[2];
4023         struct regarray *r;
4024         struct field *fidx;
4025         uint32_t idx_struct_id, idx_val;
4026
4027         CHECK(n_tokens == 3, EINVAL);
4028
4029         r = regarray_find(p, regarray);
4030         CHECK(r, EINVAL);
4031
4032         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4033         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4034         if (fidx) {
4035                 CHECK(!fidx->var_size, EINVAL);
4036
4037                 instr->type = INSTR_REGPREFETCH_RM;
4038                 if (idx[0] == 'h')
4039                         instr->type = INSTR_REGPREFETCH_RH;
4040
4041                 instr->regarray.regarray_id = r->id;
4042                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4043                 instr->regarray.idx.n_bits = fidx->n_bits;
4044                 instr->regarray.idx.offset = fidx->offset / 8;
4045                 instr->regarray.dstsrc_val = 0; /* Unused. */
4046                 return 0;
4047         }
4048
4049         /* REGPREFETCH_RI. */
4050         idx_val = strtoul(idx, &idx, 0);
4051         CHECK(!idx[0], EINVAL);
4052
4053         instr->type = INSTR_REGPREFETCH_RI;
4054         instr->regarray.regarray_id = r->id;
4055         instr->regarray.idx_val = idx_val;
4056         instr->regarray.dstsrc_val = 0; /* Unused. */
4057         return 0;
4058 }
4059
4060 static int
4061 instr_regrd_translate(struct rte_swx_pipeline *p,
4062                       struct action *action,
4063                       char **tokens,
4064                       int n_tokens,
4065                       struct instruction *instr,
4066                       struct instruction_data *data __rte_unused)
4067 {
4068         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4069         struct regarray *r;
4070         struct field *fdst, *fidx;
4071         uint32_t dst_struct_id, idx_struct_id, idx_val;
4072
4073         CHECK(n_tokens == 4, EINVAL);
4074
4075         r = regarray_find(p, regarray);
4076         CHECK(r, EINVAL);
4077
4078         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4079         CHECK(fdst, EINVAL);
4080         CHECK(!fdst->var_size, EINVAL);
4081
4082         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4083         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4084         if (fidx) {
4085                 CHECK(!fidx->var_size, EINVAL);
4086
4087                 instr->type = INSTR_REGRD_MRM;
4088                 if (dst[0] == 'h' && idx[0] != 'h')
4089                         instr->type = INSTR_REGRD_HRM;
4090                 if (dst[0] != 'h' && idx[0] == 'h')
4091                         instr->type = INSTR_REGRD_MRH;
4092                 if (dst[0] == 'h' && idx[0] == 'h')
4093                         instr->type = INSTR_REGRD_HRH;
4094
4095                 instr->regarray.regarray_id = r->id;
4096                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4097                 instr->regarray.idx.n_bits = fidx->n_bits;
4098                 instr->regarray.idx.offset = fidx->offset / 8;
4099                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4100                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4101                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4102                 return 0;
4103         }
4104
4105         /* REGRD_MRI, REGRD_HRI. */
4106         idx_val = strtoul(idx, &idx, 0);
4107         CHECK(!idx[0], EINVAL);
4108
4109         instr->type = INSTR_REGRD_MRI;
4110         if (dst[0] == 'h')
4111                 instr->type = INSTR_REGRD_HRI;
4112
4113         instr->regarray.regarray_id = r->id;
4114         instr->regarray.idx_val = idx_val;
4115         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4116         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4117         instr->regarray.dstsrc.offset = fdst->offset / 8;
4118         return 0;
4119 }
4120
4121 static int
4122 instr_regwr_translate(struct rte_swx_pipeline *p,
4123                       struct action *action,
4124                       char **tokens,
4125                       int n_tokens,
4126                       struct instruction *instr,
4127                       struct instruction_data *data __rte_unused)
4128 {
4129         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4130         struct regarray *r;
4131         struct field *fidx, *fsrc;
4132         uint64_t src_val;
4133         uint32_t idx_struct_id, idx_val, src_struct_id;
4134
4135         CHECK(n_tokens == 4, EINVAL);
4136
4137         r = regarray_find(p, regarray);
4138         CHECK(r, EINVAL);
4139
4140         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4141         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4142         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4143         if (fidx && fsrc) {
4144                 CHECK(!fidx->var_size, EINVAL);
4145                 CHECK(!fsrc->var_size, EINVAL);
4146
4147                 instr->type = INSTR_REGWR_RMM;
4148                 if (idx[0] == 'h' && src[0] != 'h')
4149                         instr->type = INSTR_REGWR_RHM;
4150                 if (idx[0] != 'h' && src[0] == 'h')
4151                         instr->type = INSTR_REGWR_RMH;
4152                 if (idx[0] == 'h' && src[0] == 'h')
4153                         instr->type = INSTR_REGWR_RHH;
4154
4155                 instr->regarray.regarray_id = r->id;
4156                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4157                 instr->regarray.idx.n_bits = fidx->n_bits;
4158                 instr->regarray.idx.offset = fidx->offset / 8;
4159                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4160                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4161                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4162                 return 0;
4163         }
4164
4165         /* REGWR_RHI, REGWR_RMI. */
4166         if (fidx && !fsrc) {
4167                 CHECK(!fidx->var_size, EINVAL);
4168
4169                 src_val = strtoull(src, &src, 0);
4170                 CHECK(!src[0], EINVAL);
4171
4172                 instr->type = INSTR_REGWR_RMI;
4173                 if (idx[0] == 'h')
4174                         instr->type = INSTR_REGWR_RHI;
4175
4176                 instr->regarray.regarray_id = r->id;
4177                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4178                 instr->regarray.idx.n_bits = fidx->n_bits;
4179                 instr->regarray.idx.offset = fidx->offset / 8;
4180                 instr->regarray.dstsrc_val = src_val;
4181                 return 0;
4182         }
4183
4184         /* REGWR_RIH, REGWR_RIM. */
4185         if (!fidx && fsrc) {
4186                 idx_val = strtoul(idx, &idx, 0);
4187                 CHECK(!idx[0], EINVAL);
4188
4189                 CHECK(!fsrc->var_size, EINVAL);
4190
4191                 instr->type = INSTR_REGWR_RIM;
4192                 if (src[0] == 'h')
4193                         instr->type = INSTR_REGWR_RIH;
4194
4195                 instr->regarray.regarray_id = r->id;
4196                 instr->regarray.idx_val = idx_val;
4197                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4198                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4199                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4200                 return 0;
4201         }
4202
4203         /* REGWR_RII. */
4204         src_val = strtoull(src, &src, 0);
4205         CHECK(!src[0], EINVAL);
4206
4207         idx_val = strtoul(idx, &idx, 0);
4208         CHECK(!idx[0], EINVAL);
4209
4210         instr->type = INSTR_REGWR_RII;
4211         instr->regarray.idx_val = idx_val;
4212         instr->regarray.dstsrc_val = src_val;
4213
4214         return 0;
4215 }
4216
4217 static int
4218 instr_regadd_translate(struct rte_swx_pipeline *p,
4219                        struct action *action,
4220                        char **tokens,
4221                        int n_tokens,
4222                        struct instruction *instr,
4223                        struct instruction_data *data __rte_unused)
4224 {
4225         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4226         struct regarray *r;
4227         struct field *fidx, *fsrc;
4228         uint64_t src_val;
4229         uint32_t idx_struct_id, idx_val, src_struct_id;
4230
4231         CHECK(n_tokens == 4, EINVAL);
4232
4233         r = regarray_find(p, regarray);
4234         CHECK(r, EINVAL);
4235
4236         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4237         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4238         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4239         if (fidx && fsrc) {
4240                 CHECK(!fidx->var_size, EINVAL);
4241                 CHECK(!fsrc->var_size, EINVAL);
4242
4243                 instr->type = INSTR_REGADD_RMM;
4244                 if (idx[0] == 'h' && src[0] != 'h')
4245                         instr->type = INSTR_REGADD_RHM;
4246                 if (idx[0] != 'h' && src[0] == 'h')
4247                         instr->type = INSTR_REGADD_RMH;
4248                 if (idx[0] == 'h' && src[0] == 'h')
4249                         instr->type = INSTR_REGADD_RHH;
4250
4251                 instr->regarray.regarray_id = r->id;
4252                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4253                 instr->regarray.idx.n_bits = fidx->n_bits;
4254                 instr->regarray.idx.offset = fidx->offset / 8;
4255                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4256                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4257                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4258                 return 0;
4259         }
4260
4261         /* REGADD_RHI, REGADD_RMI. */
4262         if (fidx && !fsrc) {
4263                 CHECK(!fidx->var_size, EINVAL);
4264
4265                 src_val = strtoull(src, &src, 0);
4266                 CHECK(!src[0], EINVAL);
4267
4268                 instr->type = INSTR_REGADD_RMI;
4269                 if (idx[0] == 'h')
4270                         instr->type = INSTR_REGADD_RHI;
4271
4272                 instr->regarray.regarray_id = r->id;
4273                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4274                 instr->regarray.idx.n_bits = fidx->n_bits;
4275                 instr->regarray.idx.offset = fidx->offset / 8;
4276                 instr->regarray.dstsrc_val = src_val;
4277                 return 0;
4278         }
4279
4280         /* REGADD_RIH, REGADD_RIM. */
4281         if (!fidx && fsrc) {
4282                 idx_val = strtoul(idx, &idx, 0);
4283                 CHECK(!idx[0], EINVAL);
4284
4285                 CHECK(!fsrc->var_size, EINVAL);
4286
4287                 instr->type = INSTR_REGADD_RIM;
4288                 if (src[0] == 'h')
4289                         instr->type = INSTR_REGADD_RIH;
4290
4291                 instr->regarray.regarray_id = r->id;
4292                 instr->regarray.idx_val = idx_val;
4293                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4294                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4295                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4296                 return 0;
4297         }
4298
4299         /* REGADD_RII. */
4300         src_val = strtoull(src, &src, 0);
4301         CHECK(!src[0], EINVAL);
4302
4303         idx_val = strtoul(idx, &idx, 0);
4304         CHECK(!idx[0], EINVAL);
4305
4306         instr->type = INSTR_REGADD_RII;
4307         instr->regarray.idx_val = idx_val;
4308         instr->regarray.dstsrc_val = src_val;
4309         return 0;
4310 }
4311
4312 static inline uint64_t *
4313 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4314 {
4315         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4316         return r->regarray;
4317 }
4318
4319 static inline uint64_t
4320 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4321 {
4322         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4323
4324         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4325         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4326         uint64_t idx64 = *idx64_ptr;
4327         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4328         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4329
4330         return idx;
4331 }
4332
4333 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4334
4335 static inline uint64_t
4336 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4337 {
4338         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4339
4340         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4341         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4342         uint64_t idx64 = *idx64_ptr;
4343         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4344
4345         return idx;
4346 }
4347
4348 #else
4349
4350 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4351
4352 #endif
4353
4354 static inline uint64_t
4355 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4356 {
4357         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4358
4359         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4360
4361         return idx;
4362 }
4363
4364 static inline uint64_t
4365 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4366 {
4367         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4368         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4369         uint64_t src64 = *src64_ptr;
4370         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4371         uint64_t src = src64 & src64_mask;
4372
4373         return src;
4374 }
4375
4376 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4377
4378 static inline uint64_t
4379 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4380 {
4381         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4382         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4383         uint64_t src64 = *src64_ptr;
4384         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4385
4386         return src;
4387 }
4388
4389 #else
4390
4391 #define instr_regarray_src_nbo instr_regarray_src_hbo
4392
4393 #endif
4394
4395 static inline void
4396 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4397 {
4398         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4399         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4400         uint64_t dst64 = *dst64_ptr;
4401         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4402
4403         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4404
4405 }
4406
4407 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4408
4409 static inline void
4410 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4411 {
4412         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4413         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4414         uint64_t dst64 = *dst64_ptr;
4415         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4416
4417         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4418         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4419 }
4420
4421 #else
4422
4423 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4424
4425 #endif
4426
4427 static inline void
4428 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4429 {
4430         struct thread *t = &p->threads[p->thread_id];
4431         struct instruction *ip = t->ip;
4432         uint64_t *regarray, idx;
4433
4434         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4435
4436         /* Structs. */
4437         regarray = instr_regarray_regarray(p, ip);
4438         idx = instr_regarray_idx_nbo(p, t, ip);
4439         rte_prefetch0(&regarray[idx]);
4440
4441         /* Thread. */
4442         thread_ip_inc(p);
4443 }
4444
4445 static inline void
4446 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4447 {
4448         struct thread *t = &p->threads[p->thread_id];
4449         struct instruction *ip = t->ip;
4450         uint64_t *regarray, idx;
4451
4452         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4453
4454         /* Structs. */
4455         regarray = instr_regarray_regarray(p, ip);
4456         idx = instr_regarray_idx_hbo(p, t, ip);
4457         rte_prefetch0(&regarray[idx]);
4458
4459         /* Thread. */
4460         thread_ip_inc(p);
4461 }
4462
4463 static inline void
4464 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4465 {
4466         struct thread *t = &p->threads[p->thread_id];
4467         struct instruction *ip = t->ip;
4468         uint64_t *regarray, idx;
4469
4470         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4471
4472         /* Structs. */
4473         regarray = instr_regarray_regarray(p, ip);
4474         idx = instr_regarray_idx_imm(p, ip);
4475         rte_prefetch0(&regarray[idx]);
4476
4477         /* Thread. */
4478         thread_ip_inc(p);
4479 }
4480
4481 static inline void
4482 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4483 {
4484         struct thread *t = &p->threads[p->thread_id];
4485         struct instruction *ip = t->ip;
4486         uint64_t *regarray, idx;
4487
4488         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4489
4490         /* Structs. */
4491         regarray = instr_regarray_regarray(p, ip);
4492         idx = instr_regarray_idx_nbo(p, t, ip);
4493         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4494
4495         /* Thread. */
4496         thread_ip_inc(p);
4497 }
4498
4499 static inline void
4500 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4501 {
4502         struct thread *t = &p->threads[p->thread_id];
4503         struct instruction *ip = t->ip;
4504         uint64_t *regarray, idx;
4505
4506         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4507
4508         /* Structs. */
4509         regarray = instr_regarray_regarray(p, ip);
4510         idx = instr_regarray_idx_hbo(p, t, ip);
4511         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4512
4513         /* Thread. */
4514         thread_ip_inc(p);
4515 }
4516
4517 static inline void
4518 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4519 {
4520         struct thread *t = &p->threads[p->thread_id];
4521         struct instruction *ip = t->ip;
4522         uint64_t *regarray, idx;
4523
4524         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4525
4526         /* Structs. */
4527         regarray = instr_regarray_regarray(p, ip);
4528         idx = instr_regarray_idx_nbo(p, t, ip);
4529         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4530
4531         /* Thread. */
4532         thread_ip_inc(p);
4533 }
4534
4535 static inline void
4536 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4537 {
4538         struct thread *t = &p->threads[p->thread_id];
4539         struct instruction *ip = t->ip;
4540         uint64_t *regarray, idx;
4541
4542         /* Structs. */
4543         regarray = instr_regarray_regarray(p, ip);
4544         idx = instr_regarray_idx_hbo(p, t, ip);
4545         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4546
4547         /* Thread. */
4548         thread_ip_inc(p);
4549 }
4550
4551 static inline void
4552 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4553 {
4554         struct thread *t = &p->threads[p->thread_id];
4555         struct instruction *ip = t->ip;
4556         uint64_t *regarray, idx;
4557
4558         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4559
4560         /* Structs. */
4561         regarray = instr_regarray_regarray(p, ip);
4562         idx = instr_regarray_idx_imm(p, ip);
4563         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4564
4565         /* Thread. */
4566         thread_ip_inc(p);
4567 }
4568
4569 static inline void
4570 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4571 {
4572         struct thread *t = &p->threads[p->thread_id];
4573         struct instruction *ip = t->ip;
4574         uint64_t *regarray, idx;
4575
4576         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4577
4578         /* Structs. */
4579         regarray = instr_regarray_regarray(p, ip);
4580         idx = instr_regarray_idx_imm(p, ip);
4581         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4582
4583         /* Thread. */
4584         thread_ip_inc(p);
4585 }
4586
4587 static inline void
4588 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4589 {
4590         struct thread *t = &p->threads[p->thread_id];
4591         struct instruction *ip = t->ip;
4592         uint64_t *regarray, idx, src;
4593
4594         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4595
4596         /* Structs. */
4597         regarray = instr_regarray_regarray(p, ip);
4598         idx = instr_regarray_idx_nbo(p, t, ip);
4599         src = instr_regarray_src_nbo(t, ip);
4600         regarray[idx] = src;
4601
4602         /* Thread. */
4603         thread_ip_inc(p);
4604 }
4605
4606 static inline void
4607 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4608 {
4609         struct thread *t = &p->threads[p->thread_id];
4610         struct instruction *ip = t->ip;
4611         uint64_t *regarray, idx, src;
4612
4613         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4614
4615         /* Structs. */
4616         regarray = instr_regarray_regarray(p, ip);
4617         idx = instr_regarray_idx_nbo(p, t, ip);
4618         src = instr_regarray_src_hbo(t, ip);
4619         regarray[idx] = src;
4620
4621         /* Thread. */
4622         thread_ip_inc(p);
4623 }
4624
4625 static inline void
4626 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4627 {
4628         struct thread *t = &p->threads[p->thread_id];
4629         struct instruction *ip = t->ip;
4630         uint64_t *regarray, idx, src;
4631
4632         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4633
4634         /* Structs. */
4635         regarray = instr_regarray_regarray(p, ip);
4636         idx = instr_regarray_idx_hbo(p, t, ip);
4637         src = instr_regarray_src_nbo(t, ip);
4638         regarray[idx] = src;
4639
4640         /* Thread. */
4641         thread_ip_inc(p);
4642 }
4643
4644 static inline void
4645 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4646 {
4647         struct thread *t = &p->threads[p->thread_id];
4648         struct instruction *ip = t->ip;
4649         uint64_t *regarray, idx, src;
4650
4651         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4652
4653         /* Structs. */
4654         regarray = instr_regarray_regarray(p, ip);
4655         idx = instr_regarray_idx_hbo(p, t, ip);
4656         src = instr_regarray_src_hbo(t, ip);
4657         regarray[idx] = src;
4658
4659         /* Thread. */
4660         thread_ip_inc(p);
4661 }
4662
4663 static inline void
4664 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4665 {
4666         struct thread *t = &p->threads[p->thread_id];
4667         struct instruction *ip = t->ip;
4668         uint64_t *regarray, idx, src;
4669
4670         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4671
4672         /* Structs. */
4673         regarray = instr_regarray_regarray(p, ip);
4674         idx = instr_regarray_idx_nbo(p, t, ip);
4675         src = ip->regarray.dstsrc_val;
4676         regarray[idx] = src;
4677
4678         /* Thread. */
4679         thread_ip_inc(p);
4680 }
4681
4682 static inline void
4683 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4684 {
4685         struct thread *t = &p->threads[p->thread_id];
4686         struct instruction *ip = t->ip;
4687         uint64_t *regarray, idx, src;
4688
4689         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4690
4691         /* Structs. */
4692         regarray = instr_regarray_regarray(p, ip);
4693         idx = instr_regarray_idx_hbo(p, t, ip);
4694         src = ip->regarray.dstsrc_val;
4695         regarray[idx] = src;
4696
4697         /* Thread. */
4698         thread_ip_inc(p);
4699 }
4700
4701 static inline void
4702 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4703 {
4704         struct thread *t = &p->threads[p->thread_id];
4705         struct instruction *ip = t->ip;
4706         uint64_t *regarray, idx, src;
4707
4708         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4709
4710         /* Structs. */
4711         regarray = instr_regarray_regarray(p, ip);
4712         idx = instr_regarray_idx_imm(p, ip);
4713         src = instr_regarray_src_nbo(t, ip);
4714         regarray[idx] = src;
4715
4716         /* Thread. */
4717         thread_ip_inc(p);
4718 }
4719
4720 static inline void
4721 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4722 {
4723         struct thread *t = &p->threads[p->thread_id];
4724         struct instruction *ip = t->ip;
4725         uint64_t *regarray, idx, src;
4726
4727         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4728
4729         /* Structs. */
4730         regarray = instr_regarray_regarray(p, ip);
4731         idx = instr_regarray_idx_imm(p, ip);
4732         src = instr_regarray_src_hbo(t, ip);
4733         regarray[idx] = src;
4734
4735         /* Thread. */
4736         thread_ip_inc(p);
4737 }
4738
4739 static inline void
4740 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4741 {
4742         struct thread *t = &p->threads[p->thread_id];
4743         struct instruction *ip = t->ip;
4744         uint64_t *regarray, idx, src;
4745
4746         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4747
4748         /* Structs. */
4749         regarray = instr_regarray_regarray(p, ip);
4750         idx = instr_regarray_idx_imm(p, ip);
4751         src = ip->regarray.dstsrc_val;
4752         regarray[idx] = src;
4753
4754         /* Thread. */
4755         thread_ip_inc(p);
4756 }
4757
4758 static inline void
4759 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4760 {
4761         struct thread *t = &p->threads[p->thread_id];
4762         struct instruction *ip = t->ip;
4763         uint64_t *regarray, idx, src;
4764
4765         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4766
4767         /* Structs. */
4768         regarray = instr_regarray_regarray(p, ip);
4769         idx = instr_regarray_idx_nbo(p, t, ip);
4770         src = instr_regarray_src_nbo(t, ip);
4771         regarray[idx] += src;
4772
4773         /* Thread. */
4774         thread_ip_inc(p);
4775 }
4776
4777 static inline void
4778 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4779 {
4780         struct thread *t = &p->threads[p->thread_id];
4781         struct instruction *ip = t->ip;
4782         uint64_t *regarray, idx, src;
4783
4784         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4785
4786         /* Structs. */
4787         regarray = instr_regarray_regarray(p, ip);
4788         idx = instr_regarray_idx_nbo(p, t, ip);
4789         src = instr_regarray_src_hbo(t, ip);
4790         regarray[idx] += src;
4791
4792         /* Thread. */
4793         thread_ip_inc(p);
4794 }
4795
4796 static inline void
4797 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4798 {
4799         struct thread *t = &p->threads[p->thread_id];
4800         struct instruction *ip = t->ip;
4801         uint64_t *regarray, idx, src;
4802
4803         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4804
4805         /* Structs. */
4806         regarray = instr_regarray_regarray(p, ip);
4807         idx = instr_regarray_idx_hbo(p, t, ip);
4808         src = instr_regarray_src_nbo(t, ip);
4809         regarray[idx] += src;
4810
4811         /* Thread. */
4812         thread_ip_inc(p);
4813 }
4814
4815 static inline void
4816 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4817 {
4818         struct thread *t = &p->threads[p->thread_id];
4819         struct instruction *ip = t->ip;
4820         uint64_t *regarray, idx, src;
4821
4822         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4823
4824         /* Structs. */
4825         regarray = instr_regarray_regarray(p, ip);
4826         idx = instr_regarray_idx_hbo(p, t, ip);
4827         src = instr_regarray_src_hbo(t, ip);
4828         regarray[idx] += src;
4829
4830         /* Thread. */
4831         thread_ip_inc(p);
4832 }
4833
4834 static inline void
4835 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4836 {
4837         struct thread *t = &p->threads[p->thread_id];
4838         struct instruction *ip = t->ip;
4839         uint64_t *regarray, idx, src;
4840
4841         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4842
4843         /* Structs. */
4844         regarray = instr_regarray_regarray(p, ip);
4845         idx = instr_regarray_idx_nbo(p, t, ip);
4846         src = ip->regarray.dstsrc_val;
4847         regarray[idx] += src;
4848
4849         /* Thread. */
4850         thread_ip_inc(p);
4851 }
4852
4853 static inline void
4854 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4855 {
4856         struct thread *t = &p->threads[p->thread_id];
4857         struct instruction *ip = t->ip;
4858         uint64_t *regarray, idx, src;
4859
4860         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4861
4862         /* Structs. */
4863         regarray = instr_regarray_regarray(p, ip);
4864         idx = instr_regarray_idx_hbo(p, t, ip);
4865         src = ip->regarray.dstsrc_val;
4866         regarray[idx] += src;
4867
4868         /* Thread. */
4869         thread_ip_inc(p);
4870 }
4871
4872 static inline void
4873 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4874 {
4875         struct thread *t = &p->threads[p->thread_id];
4876         struct instruction *ip = t->ip;
4877         uint64_t *regarray, idx, src;
4878
4879         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4880
4881         /* Structs. */
4882         regarray = instr_regarray_regarray(p, ip);
4883         idx = instr_regarray_idx_imm(p, ip);
4884         src = instr_regarray_src_nbo(t, ip);
4885         regarray[idx] += src;
4886
4887         /* Thread. */
4888         thread_ip_inc(p);
4889 }
4890
4891 static inline void
4892 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4893 {
4894         struct thread *t = &p->threads[p->thread_id];
4895         struct instruction *ip = t->ip;
4896         uint64_t *regarray, idx, src;
4897
4898         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4899
4900         /* Structs. */
4901         regarray = instr_regarray_regarray(p, ip);
4902         idx = instr_regarray_idx_imm(p, ip);
4903         src = instr_regarray_src_hbo(t, ip);
4904         regarray[idx] += src;
4905
4906         /* Thread. */
4907         thread_ip_inc(p);
4908 }
4909
4910 static inline void
4911 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4912 {
4913         struct thread *t = &p->threads[p->thread_id];
4914         struct instruction *ip = t->ip;
4915         uint64_t *regarray, idx, src;
4916
4917         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4918
4919         /* Structs. */
4920         regarray = instr_regarray_regarray(p, ip);
4921         idx = instr_regarray_idx_imm(p, ip);
4922         src = ip->regarray.dstsrc_val;
4923         regarray[idx] += src;
4924
4925         /* Thread. */
4926         thread_ip_inc(p);
4927 }
4928
4929 /*
4930  * metarray.
4931  */
4932 static struct metarray *
4933 metarray_find(struct rte_swx_pipeline *p, const char *name);
4934
4935 static int
4936 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4937                             struct action *action,
4938                             char **tokens,
4939                             int n_tokens,
4940                             struct instruction *instr,
4941                             struct instruction_data *data __rte_unused)
4942 {
4943         char *metarray = tokens[1], *idx = tokens[2];
4944         struct metarray *m;
4945         struct field *fidx;
4946         uint32_t idx_struct_id, idx_val;
4947
4948         CHECK(n_tokens == 3, EINVAL);
4949
4950         m = metarray_find(p, metarray);
4951         CHECK(m, EINVAL);
4952
4953         /* METPREFETCH_H, METPREFETCH_M. */
4954         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4955         if (fidx) {
4956                 CHECK(!fidx->var_size, EINVAL);
4957
4958                 instr->type = INSTR_METPREFETCH_M;
4959                 if (idx[0] == 'h')
4960                         instr->type = INSTR_METPREFETCH_H;
4961
4962                 instr->meter.metarray_id = m->id;
4963                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4964                 instr->meter.idx.n_bits = fidx->n_bits;
4965                 instr->meter.idx.offset = fidx->offset / 8;
4966                 return 0;
4967         }
4968
4969         /* METPREFETCH_I. */
4970         idx_val = strtoul(idx, &idx, 0);
4971         CHECK(!idx[0], EINVAL);
4972
4973         instr->type = INSTR_METPREFETCH_I;
4974         instr->meter.metarray_id = m->id;
4975         instr->meter.idx_val = idx_val;
4976         return 0;
4977 }
4978
4979 static int
4980 instr_meter_translate(struct rte_swx_pipeline *p,
4981                       struct action *action,
4982                       char **tokens,
4983                       int n_tokens,
4984                       struct instruction *instr,
4985                       struct instruction_data *data __rte_unused)
4986 {
4987         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4988         char *color_in = tokens[4], *color_out = tokens[5];
4989         struct metarray *m;
4990         struct field *fidx, *flength, *fcin, *fcout;
4991         uint32_t idx_struct_id, length_struct_id;
4992         uint32_t color_in_struct_id, color_out_struct_id;
4993
4994         CHECK(n_tokens == 6, EINVAL);
4995
4996         m = metarray_find(p, metarray);
4997         CHECK(m, EINVAL);
4998
4999         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5000
5001         flength = struct_field_parse(p, action, length, &length_struct_id);
5002         CHECK(flength, EINVAL);
5003         CHECK(!flength->var_size, EINVAL);
5004
5005         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5006
5007         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5008         CHECK(fcout, EINVAL);
5009         CHECK(!fcout->var_size, EINVAL);
5010
5011         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5012         if (fidx && fcin) {
5013                 CHECK(!fidx->var_size, EINVAL);
5014                 CHECK(!fcin->var_size, EINVAL);
5015
5016                 instr->type = INSTR_METER_MMM;
5017                 if (idx[0] == 'h' && length[0] == 'h')
5018                         instr->type = INSTR_METER_HHM;
5019                 if (idx[0] == 'h' && length[0] != 'h')
5020                         instr->type = INSTR_METER_HMM;
5021                 if (idx[0] != 'h' && length[0] == 'h')
5022                         instr->type = INSTR_METER_MHM;
5023
5024                 instr->meter.metarray_id = m->id;
5025
5026                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5027                 instr->meter.idx.n_bits = fidx->n_bits;
5028                 instr->meter.idx.offset = fidx->offset / 8;
5029
5030                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5031                 instr->meter.length.n_bits = flength->n_bits;
5032                 instr->meter.length.offset = flength->offset / 8;
5033
5034                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5035                 instr->meter.color_in.n_bits = fcin->n_bits;
5036                 instr->meter.color_in.offset = fcin->offset / 8;
5037
5038                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5039                 instr->meter.color_out.n_bits = fcout->n_bits;
5040                 instr->meter.color_out.offset = fcout->offset / 8;
5041
5042                 return 0;
5043         }
5044
5045         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5046         if (fidx && !fcin) {
5047                 uint32_t color_in_val;
5048
5049                 CHECK(!fidx->var_size, EINVAL);
5050
5051                 color_in_val = strtoul(color_in, &color_in, 0);
5052                 CHECK(!color_in[0], EINVAL);
5053
5054                 instr->type = INSTR_METER_MMI;
5055                 if (idx[0] == 'h' && length[0] == 'h')
5056                         instr->type = INSTR_METER_HHI;
5057                 if (idx[0] == 'h' && length[0] != 'h')
5058                         instr->type = INSTR_METER_HMI;
5059                 if (idx[0] != 'h' && length[0] == 'h')
5060                         instr->type = INSTR_METER_MHI;
5061
5062                 instr->meter.metarray_id = m->id;
5063
5064                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5065                 instr->meter.idx.n_bits = fidx->n_bits;
5066                 instr->meter.idx.offset = fidx->offset / 8;
5067
5068                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5069                 instr->meter.length.n_bits = flength->n_bits;
5070                 instr->meter.length.offset = flength->offset / 8;
5071
5072                 instr->meter.color_in_val = color_in_val;
5073
5074                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5075                 instr->meter.color_out.n_bits = fcout->n_bits;
5076                 instr->meter.color_out.offset = fcout->offset / 8;
5077
5078                 return 0;
5079         }
5080
5081         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5082         if (!fidx && fcin) {
5083                 uint32_t idx_val;
5084
5085                 idx_val = strtoul(idx, &idx, 0);
5086                 CHECK(!idx[0], EINVAL);
5087
5088                 CHECK(!fcin->var_size, EINVAL);
5089
5090                 instr->type = INSTR_METER_IMM;
5091                 if (length[0] == 'h')
5092                         instr->type = INSTR_METER_IHM;
5093
5094                 instr->meter.metarray_id = m->id;
5095
5096                 instr->meter.idx_val = idx_val;
5097
5098                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5099                 instr->meter.length.n_bits = flength->n_bits;
5100                 instr->meter.length.offset = flength->offset / 8;
5101
5102                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5103                 instr->meter.color_in.n_bits = fcin->n_bits;
5104                 instr->meter.color_in.offset = fcin->offset / 8;
5105
5106                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5107                 instr->meter.color_out.n_bits = fcout->n_bits;
5108                 instr->meter.color_out.offset = fcout->offset / 8;
5109
5110                 return 0;
5111         }
5112
5113         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5114         if (!fidx && !fcin) {
5115                 uint32_t idx_val, color_in_val;
5116
5117                 idx_val = strtoul(idx, &idx, 0);
5118                 CHECK(!idx[0], EINVAL);
5119
5120                 color_in_val = strtoul(color_in, &color_in, 0);
5121                 CHECK(!color_in[0], EINVAL);
5122
5123                 instr->type = INSTR_METER_IMI;
5124                 if (length[0] == 'h')
5125                         instr->type = INSTR_METER_IHI;
5126
5127                 instr->meter.metarray_id = m->id;
5128
5129                 instr->meter.idx_val = idx_val;
5130
5131                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5132                 instr->meter.length.n_bits = flength->n_bits;
5133                 instr->meter.length.offset = flength->offset / 8;
5134
5135                 instr->meter.color_in_val = color_in_val;
5136
5137                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5138                 instr->meter.color_out.n_bits = fcout->n_bits;
5139                 instr->meter.color_out.offset = fcout->offset / 8;
5140
5141                 return 0;
5142         }
5143
5144         CHECK(0, EINVAL);
5145 }
5146
5147 static inline struct meter *
5148 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5149 {
5150         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5151
5152         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5153         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5154         uint64_t idx64 = *idx64_ptr;
5155         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5156         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5157
5158         return &r->metarray[idx];
5159 }
5160
5161 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5162
5163 static inline struct meter *
5164 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5165 {
5166         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5167
5168         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5169         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5170         uint64_t idx64 = *idx64_ptr;
5171         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5172
5173         return &r->metarray[idx];
5174 }
5175
5176 #else
5177
5178 #define instr_meter_idx_nbo instr_meter_idx_hbo
5179
5180 #endif
5181
5182 static inline struct meter *
5183 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5184 {
5185         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5186
5187         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5188
5189         return &r->metarray[idx];
5190 }
5191
5192 static inline uint32_t
5193 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5194 {
5195         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5196         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5197         uint64_t src64 = *src64_ptr;
5198         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5199         uint64_t src = src64 & src64_mask;
5200
5201         return (uint32_t)src;
5202 }
5203
5204 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5205
5206 static inline uint32_t
5207 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5208 {
5209         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5210         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5211         uint64_t src64 = *src64_ptr;
5212         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5213
5214         return (uint32_t)src;
5215 }
5216
5217 #else
5218
5219 #define instr_meter_length_nbo instr_meter_length_hbo
5220
5221 #endif
5222
5223 static inline enum rte_color
5224 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5225 {
5226         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5227         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5228         uint64_t src64 = *src64_ptr;
5229         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5230         uint64_t src = src64 & src64_mask;
5231
5232         return (enum rte_color)src;
5233 }
5234
5235 static inline void
5236 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5237 {
5238         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5239         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5240         uint64_t dst64 = *dst64_ptr;
5241         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5242
5243         uint64_t src = (uint64_t)color_out;
5244
5245         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5246 }
5247
5248 static inline void
5249 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5250 {
5251         struct thread *t = &p->threads[p->thread_id];
5252         struct instruction *ip = t->ip;
5253         struct meter *m;
5254
5255         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5256
5257         /* Structs. */
5258         m = instr_meter_idx_nbo(p, t, ip);
5259         rte_prefetch0(m);
5260
5261         /* Thread. */
5262         thread_ip_inc(p);
5263 }
5264
5265 static inline void
5266 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5267 {
5268         struct thread *t = &p->threads[p->thread_id];
5269         struct instruction *ip = t->ip;
5270         struct meter *m;
5271
5272         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5273
5274         /* Structs. */
5275         m = instr_meter_idx_hbo(p, t, ip);
5276         rte_prefetch0(m);
5277
5278         /* Thread. */
5279         thread_ip_inc(p);
5280 }
5281
5282 static inline void
5283 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5284 {
5285         struct thread *t = &p->threads[p->thread_id];
5286         struct instruction *ip = t->ip;
5287         struct meter *m;
5288
5289         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5290
5291         /* Structs. */
5292         m = instr_meter_idx_imm(p, ip);
5293         rte_prefetch0(m);
5294
5295         /* Thread. */
5296         thread_ip_inc(p);
5297 }
5298
5299 static inline void
5300 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5301 {
5302         struct thread *t = &p->threads[p->thread_id];
5303         struct instruction *ip = t->ip;
5304         struct meter *m;
5305         uint64_t time, n_pkts, n_bytes;
5306         uint32_t length;
5307         enum rte_color color_in, color_out;
5308
5309         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5310
5311         /* Structs. */
5312         m = instr_meter_idx_nbo(p, t, ip);
5313         rte_prefetch0(m->n_pkts);
5314         time = rte_get_tsc_cycles();
5315         length = instr_meter_length_nbo(t, ip);
5316         color_in = instr_meter_color_in_hbo(t, ip);
5317
5318         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5319                 &m->profile->profile,
5320                 time,
5321                 length,
5322                 color_in);
5323
5324         color_out &= m->color_mask;
5325
5326         n_pkts = m->n_pkts[color_out];
5327         n_bytes = m->n_bytes[color_out];
5328
5329         instr_meter_color_out_hbo_set(t, ip, color_out);
5330
5331         m->n_pkts[color_out] = n_pkts + 1;
5332         m->n_bytes[color_out] = n_bytes + length;
5333
5334         /* Thread. */
5335         thread_ip_inc(p);
5336 }
5337
5338 static inline void
5339 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5340 {
5341         struct thread *t = &p->threads[p->thread_id];
5342         struct instruction *ip = t->ip;
5343         struct meter *m;
5344         uint64_t time, n_pkts, n_bytes;
5345         uint32_t length;
5346         enum rte_color color_in, color_out;
5347
5348         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5349
5350         /* Structs. */
5351         m = instr_meter_idx_nbo(p, t, ip);
5352         rte_prefetch0(m->n_pkts);
5353         time = rte_get_tsc_cycles();
5354         length = instr_meter_length_nbo(t, ip);
5355         color_in = (enum rte_color)ip->meter.color_in_val;
5356
5357         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5358                 &m->profile->profile,
5359                 time,
5360                 length,
5361                 color_in);
5362
5363         color_out &= m->color_mask;
5364
5365         n_pkts = m->n_pkts[color_out];
5366         n_bytes = m->n_bytes[color_out];
5367
5368         instr_meter_color_out_hbo_set(t, ip, color_out);
5369
5370         m->n_pkts[color_out] = n_pkts + 1;
5371         m->n_bytes[color_out] = n_bytes + length;
5372
5373         /* Thread. */
5374         thread_ip_inc(p);
5375 }
5376
5377 static inline void
5378 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5379 {
5380         struct thread *t = &p->threads[p->thread_id];
5381         struct instruction *ip = t->ip;
5382         struct meter *m;
5383         uint64_t time, n_pkts, n_bytes;
5384         uint32_t length;
5385         enum rte_color color_in, color_out;
5386
5387         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5388
5389         /* Structs. */
5390         m = instr_meter_idx_nbo(p, t, ip);
5391         rte_prefetch0(m->n_pkts);
5392         time = rte_get_tsc_cycles();
5393         length = instr_meter_length_hbo(t, ip);
5394         color_in = instr_meter_color_in_hbo(t, ip);
5395
5396         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5397                 &m->profile->profile,
5398                 time,
5399                 length,
5400                 color_in);
5401
5402         color_out &= m->color_mask;
5403
5404         n_pkts = m->n_pkts[color_out];
5405         n_bytes = m->n_bytes[color_out];
5406
5407         instr_meter_color_out_hbo_set(t, ip, color_out);
5408
5409         m->n_pkts[color_out] = n_pkts + 1;
5410         m->n_bytes[color_out] = n_bytes + length;
5411
5412         /* Thread. */
5413         thread_ip_inc(p);
5414 }
5415 static inline void
5416 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5417 {
5418         struct thread *t = &p->threads[p->thread_id];
5419         struct instruction *ip = t->ip;
5420         struct meter *m;
5421         uint64_t time, n_pkts, n_bytes;
5422         uint32_t length;
5423         enum rte_color color_in, color_out;
5424
5425         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5426
5427         /* Structs. */
5428         m = instr_meter_idx_nbo(p, t, ip);
5429         rte_prefetch0(m->n_pkts);
5430         time = rte_get_tsc_cycles();
5431         length = instr_meter_length_hbo(t, ip);
5432         color_in = (enum rte_color)ip->meter.color_in_val;
5433
5434         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5435                 &m->profile->profile,
5436                 time,
5437                 length,
5438                 color_in);
5439
5440         color_out &= m->color_mask;
5441
5442         n_pkts = m->n_pkts[color_out];
5443         n_bytes = m->n_bytes[color_out];
5444
5445         instr_meter_color_out_hbo_set(t, ip, color_out);
5446
5447         m->n_pkts[color_out] = n_pkts + 1;
5448         m->n_bytes[color_out] = n_bytes + length;
5449
5450         /* Thread. */
5451         thread_ip_inc(p);
5452 }
5453
5454 static inline void
5455 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5456 {
5457         struct thread *t = &p->threads[p->thread_id];
5458         struct instruction *ip = t->ip;
5459         struct meter *m;
5460         uint64_t time, n_pkts, n_bytes;
5461         uint32_t length;
5462         enum rte_color color_in, color_out;
5463
5464         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5465
5466         /* Structs. */
5467         m = instr_meter_idx_hbo(p, t, ip);
5468         rte_prefetch0(m->n_pkts);
5469         time = rte_get_tsc_cycles();
5470         length = instr_meter_length_nbo(t, ip);
5471         color_in = instr_meter_color_in_hbo(t, ip);
5472
5473         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5474                 &m->profile->profile,
5475                 time,
5476                 length,
5477                 color_in);
5478
5479         color_out &= m->color_mask;
5480
5481         n_pkts = m->n_pkts[color_out];
5482         n_bytes = m->n_bytes[color_out];
5483
5484         instr_meter_color_out_hbo_set(t, ip, color_out);
5485
5486         m->n_pkts[color_out] = n_pkts + 1;
5487         m->n_bytes[color_out] = n_bytes + length;
5488
5489         /* Thread. */
5490         thread_ip_inc(p);
5491 }
5492
5493 static inline void
5494 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5495 {
5496         struct thread *t = &p->threads[p->thread_id];
5497         struct instruction *ip = t->ip;
5498         struct meter *m;
5499         uint64_t time, n_pkts, n_bytes;
5500         uint32_t length;
5501         enum rte_color color_in, color_out;
5502
5503         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5504
5505         /* Structs. */
5506         m = instr_meter_idx_hbo(p, t, ip);
5507         rte_prefetch0(m->n_pkts);
5508         time = rte_get_tsc_cycles();
5509         length = instr_meter_length_nbo(t, ip);
5510         color_in = (enum rte_color)ip->meter.color_in_val;
5511
5512         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5513                 &m->profile->profile,
5514                 time,
5515                 length,
5516                 color_in);
5517
5518         color_out &= m->color_mask;
5519
5520         n_pkts = m->n_pkts[color_out];
5521         n_bytes = m->n_bytes[color_out];
5522
5523         instr_meter_color_out_hbo_set(t, ip, color_out);
5524
5525         m->n_pkts[color_out] = n_pkts + 1;
5526         m->n_bytes[color_out] = n_bytes + length;
5527
5528         /* Thread. */
5529         thread_ip_inc(p);
5530 }
5531
5532 static inline void
5533 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5534 {
5535         struct thread *t = &p->threads[p->thread_id];
5536         struct instruction *ip = t->ip;
5537         struct meter *m;
5538         uint64_t time, n_pkts, n_bytes;
5539         uint32_t length;
5540         enum rte_color color_in, color_out;
5541
5542         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5543
5544         /* Structs. */
5545         m = instr_meter_idx_hbo(p, t, ip);
5546         rte_prefetch0(m->n_pkts);
5547         time = rte_get_tsc_cycles();
5548         length = instr_meter_length_hbo(t, ip);
5549         color_in = instr_meter_color_in_hbo(t, ip);
5550
5551         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5552                 &m->profile->profile,
5553                 time,
5554                 length,
5555                 color_in);
5556
5557         color_out &= m->color_mask;
5558
5559         n_pkts = m->n_pkts[color_out];
5560         n_bytes = m->n_bytes[color_out];
5561
5562         instr_meter_color_out_hbo_set(t, ip, color_out);
5563
5564         m->n_pkts[color_out] = n_pkts + 1;
5565         m->n_bytes[color_out] = n_bytes + length;
5566
5567         /* Thread. */
5568         thread_ip_inc(p);
5569 }
5570
5571 static inline void
5572 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5573 {
5574         struct thread *t = &p->threads[p->thread_id];
5575         struct instruction *ip = t->ip;
5576         struct meter *m;
5577         uint64_t time, n_pkts, n_bytes;
5578         uint32_t length;
5579         enum rte_color color_in, color_out;
5580
5581         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5582
5583         /* Structs. */
5584         m = instr_meter_idx_hbo(p, t, ip);
5585         rte_prefetch0(m->n_pkts);
5586         time = rte_get_tsc_cycles();
5587         length = instr_meter_length_hbo(t, ip);
5588         color_in = (enum rte_color)ip->meter.color_in_val;
5589
5590         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5591                 &m->profile->profile,
5592                 time,
5593                 length,
5594                 color_in);
5595
5596         color_out &= m->color_mask;
5597
5598         n_pkts = m->n_pkts[color_out];
5599         n_bytes = m->n_bytes[color_out];
5600
5601         instr_meter_color_out_hbo_set(t, ip, color_out);
5602
5603         m->n_pkts[color_out] = n_pkts + 1;
5604         m->n_bytes[color_out] = n_bytes + length;
5605
5606         /* Thread. */
5607         thread_ip_inc(p);
5608 }
5609
5610 static inline void
5611 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5612 {
5613         struct thread *t = &p->threads[p->thread_id];
5614         struct instruction *ip = t->ip;
5615         struct meter *m;
5616         uint64_t time, n_pkts, n_bytes;
5617         uint32_t length;
5618         enum rte_color color_in, color_out;
5619
5620         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5621
5622         /* Structs. */
5623         m = instr_meter_idx_imm(p, ip);
5624         rte_prefetch0(m->n_pkts);
5625         time = rte_get_tsc_cycles();
5626         length = instr_meter_length_nbo(t, ip);
5627         color_in = instr_meter_color_in_hbo(t, ip);
5628
5629         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5630                 &m->profile->profile,
5631                 time,
5632                 length,
5633                 color_in);
5634
5635         color_out &= m->color_mask;
5636
5637         n_pkts = m->n_pkts[color_out];
5638         n_bytes = m->n_bytes[color_out];
5639
5640         instr_meter_color_out_hbo_set(t, ip, color_out);
5641
5642         m->n_pkts[color_out] = n_pkts + 1;
5643         m->n_bytes[color_out] = n_bytes + length;
5644
5645         /* Thread. */
5646         thread_ip_inc(p);
5647 }
5648
5649 static inline void
5650 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5651 {
5652         struct thread *t = &p->threads[p->thread_id];
5653         struct instruction *ip = t->ip;
5654         struct meter *m;
5655         uint64_t time, n_pkts, n_bytes;
5656         uint32_t length;
5657         enum rte_color color_in, color_out;
5658
5659         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5660
5661         /* Structs. */
5662         m = instr_meter_idx_imm(p, ip);
5663         rte_prefetch0(m->n_pkts);
5664         time = rte_get_tsc_cycles();
5665         length = instr_meter_length_nbo(t, ip);
5666         color_in = (enum rte_color)ip->meter.color_in_val;
5667
5668         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5669                 &m->profile->profile,
5670                 time,
5671                 length,
5672                 color_in);
5673
5674         color_out &= m->color_mask;
5675
5676         n_pkts = m->n_pkts[color_out];
5677         n_bytes = m->n_bytes[color_out];
5678
5679         instr_meter_color_out_hbo_set(t, ip, color_out);
5680
5681         m->n_pkts[color_out] = n_pkts + 1;
5682         m->n_bytes[color_out] = n_bytes + length;
5683
5684         /* Thread. */
5685         thread_ip_inc(p);
5686 }
5687
5688 static inline void
5689 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5690 {
5691         struct thread *t = &p->threads[p->thread_id];
5692         struct instruction *ip = t->ip;
5693         struct meter *m;
5694         uint64_t time, n_pkts, n_bytes;
5695         uint32_t length;
5696         enum rte_color color_in, color_out;
5697
5698         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5699
5700         /* Structs. */
5701         m = instr_meter_idx_imm(p, ip);
5702         rte_prefetch0(m->n_pkts);
5703         time = rte_get_tsc_cycles();
5704         length = instr_meter_length_hbo(t, ip);
5705         color_in = instr_meter_color_in_hbo(t, ip);
5706
5707         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5708                 &m->profile->profile,
5709                 time,
5710                 length,
5711                 color_in);
5712
5713         color_out &= m->color_mask;
5714
5715         n_pkts = m->n_pkts[color_out];
5716         n_bytes = m->n_bytes[color_out];
5717
5718         instr_meter_color_out_hbo_set(t, ip, color_out);
5719
5720         m->n_pkts[color_out] = n_pkts + 1;
5721         m->n_bytes[color_out] = n_bytes + length;
5722
5723         /* Thread. */
5724         thread_ip_inc(p);
5725 }
5726 static inline void
5727 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5728 {
5729         struct thread *t = &p->threads[p->thread_id];
5730         struct instruction *ip = t->ip;
5731         struct meter *m;
5732         uint64_t time, n_pkts, n_bytes;
5733         uint32_t length;
5734         enum rte_color color_in, color_out;
5735
5736         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5737
5738         /* Structs. */
5739         m = instr_meter_idx_imm(p, ip);
5740         rte_prefetch0(m->n_pkts);
5741         time = rte_get_tsc_cycles();
5742         length = instr_meter_length_hbo(t, ip);
5743         color_in = (enum rte_color)ip->meter.color_in_val;
5744
5745         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5746                 &m->profile->profile,
5747                 time,
5748                 length,
5749                 color_in);
5750
5751         color_out &= m->color_mask;
5752
5753         n_pkts = m->n_pkts[color_out];
5754         n_bytes = m->n_bytes[color_out];
5755
5756         instr_meter_color_out_hbo_set(t, ip, color_out);
5757
5758         m->n_pkts[color_out] = n_pkts + 1;
5759         m->n_bytes[color_out] = n_bytes + length;
5760
5761         /* Thread. */
5762         thread_ip_inc(p);
5763 }
5764
5765 /*
5766  * jmp.
5767  */
5768 static int
5769 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5770                     struct action *action __rte_unused,
5771                     char **tokens,
5772                     int n_tokens,
5773                     struct instruction *instr,
5774                     struct instruction_data *data)
5775 {
5776         CHECK(n_tokens == 2, EINVAL);
5777
5778         strcpy(data->jmp_label, tokens[1]);
5779
5780         instr->type = INSTR_JMP;
5781         instr->jmp.ip = NULL; /* Resolved later. */
5782         return 0;
5783 }
5784
5785 static int
5786 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5787                           struct action *action __rte_unused,
5788                           char **tokens,
5789                           int n_tokens,
5790                           struct instruction *instr,
5791                           struct instruction_data *data)
5792 {
5793         struct header *h;
5794
5795         CHECK(n_tokens == 3, EINVAL);
5796
5797         strcpy(data->jmp_label, tokens[1]);
5798
5799         h = header_parse(p, tokens[2]);
5800         CHECK(h, EINVAL);
5801
5802         instr->type = INSTR_JMP_VALID;
5803         instr->jmp.ip = NULL; /* Resolved later. */
5804         instr->jmp.header_id = h->id;
5805         return 0;
5806 }
5807
5808 static int
5809 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5810                             struct action *action __rte_unused,
5811                             char **tokens,
5812                             int n_tokens,
5813                             struct instruction *instr,
5814                             struct instruction_data *data)
5815 {
5816         struct header *h;
5817
5818         CHECK(n_tokens == 3, EINVAL);
5819
5820         strcpy(data->jmp_label, tokens[1]);
5821
5822         h = header_parse(p, tokens[2]);
5823         CHECK(h, EINVAL);
5824
5825         instr->type = INSTR_JMP_INVALID;
5826         instr->jmp.ip = NULL; /* Resolved later. */
5827         instr->jmp.header_id = h->id;
5828         return 0;
5829 }
5830
5831 static int
5832 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5833                         struct action *action,
5834                         char **tokens,
5835                         int n_tokens,
5836                         struct instruction *instr,
5837                         struct instruction_data *data)
5838 {
5839         CHECK(!action, EINVAL);
5840         CHECK(n_tokens == 2, EINVAL);
5841
5842         strcpy(data->jmp_label, tokens[1]);
5843
5844         instr->type = INSTR_JMP_HIT;
5845         instr->jmp.ip = NULL; /* Resolved later. */
5846         return 0;
5847 }
5848
5849 static int
5850 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5851                          struct action *action,
5852                          char **tokens,
5853                          int n_tokens,
5854                          struct instruction *instr,
5855                          struct instruction_data *data)
5856 {
5857         CHECK(!action, EINVAL);
5858         CHECK(n_tokens == 2, EINVAL);
5859
5860         strcpy(data->jmp_label, tokens[1]);
5861
5862         instr->type = INSTR_JMP_MISS;
5863         instr->jmp.ip = NULL; /* Resolved later. */
5864         return 0;
5865 }
5866
5867 static int
5868 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5869                                struct action *action,
5870                                char **tokens,
5871                                int n_tokens,
5872                                struct instruction *instr,
5873                                struct instruction_data *data)
5874 {
5875         struct action *a;
5876
5877         CHECK(!action, EINVAL);
5878         CHECK(n_tokens == 3, EINVAL);
5879
5880         strcpy(data->jmp_label, tokens[1]);
5881
5882         a = action_find(p, tokens[2]);
5883         CHECK(a, EINVAL);
5884
5885         instr->type = INSTR_JMP_ACTION_HIT;
5886         instr->jmp.ip = NULL; /* Resolved later. */
5887         instr->jmp.action_id = a->id;
5888         return 0;
5889 }
5890
5891 static int
5892 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5893                                 struct action *action,
5894                                 char **tokens,
5895                                 int n_tokens,
5896                                 struct instruction *instr,
5897                                 struct instruction_data *data)
5898 {
5899         struct action *a;
5900
5901         CHECK(!action, EINVAL);
5902         CHECK(n_tokens == 3, EINVAL);
5903
5904         strcpy(data->jmp_label, tokens[1]);
5905
5906         a = action_find(p, tokens[2]);
5907         CHECK(a, EINVAL);
5908
5909         instr->type = INSTR_JMP_ACTION_MISS;
5910         instr->jmp.ip = NULL; /* Resolved later. */
5911         instr->jmp.action_id = a->id;
5912         return 0;
5913 }
5914
5915 static int
5916 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5917                        struct action *action,
5918                        char **tokens,
5919                        int n_tokens,
5920                        struct instruction *instr,
5921                        struct instruction_data *data)
5922 {
5923         char *a = tokens[2], *b = tokens[3];
5924         struct field *fa, *fb;
5925         uint64_t b_val;
5926         uint32_t a_struct_id, b_struct_id;
5927
5928         CHECK(n_tokens == 4, EINVAL);
5929
5930         strcpy(data->jmp_label, tokens[1]);
5931
5932         fa = struct_field_parse(p, action, a, &a_struct_id);
5933         CHECK(fa, EINVAL);
5934         CHECK(!fa->var_size, EINVAL);
5935
5936         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5937         fb = struct_field_parse(p, action, b, &b_struct_id);
5938         if (fb) {
5939                 CHECK(!fb->var_size, EINVAL);
5940
5941                 instr->type = INSTR_JMP_EQ;
5942                 if (a[0] != 'h' && b[0] == 'h')
5943                         instr->type = INSTR_JMP_EQ_MH;
5944                 if (a[0] == 'h' && b[0] != 'h')
5945                         instr->type = INSTR_JMP_EQ_HM;
5946                 if (a[0] == 'h' && b[0] == 'h')
5947                         instr->type = INSTR_JMP_EQ_HH;
5948                 instr->jmp.ip = NULL; /* Resolved later. */
5949
5950                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5951                 instr->jmp.a.n_bits = fa->n_bits;
5952                 instr->jmp.a.offset = fa->offset / 8;
5953                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5954                 instr->jmp.b.n_bits = fb->n_bits;
5955                 instr->jmp.b.offset = fb->offset / 8;
5956                 return 0;
5957         }
5958
5959         /* JMP_EQ_I. */
5960         b_val = strtoull(b, &b, 0);
5961         CHECK(!b[0], EINVAL);
5962
5963         if (a[0] == 'h')
5964                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5965
5966         instr->type = INSTR_JMP_EQ_I;
5967         instr->jmp.ip = NULL; /* Resolved later. */
5968         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5969         instr->jmp.a.n_bits = fa->n_bits;
5970         instr->jmp.a.offset = fa->offset / 8;
5971         instr->jmp.b_val = b_val;
5972         return 0;
5973 }
5974
5975 static int
5976 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5977                         struct action *action,
5978                         char **tokens,
5979                         int n_tokens,
5980                         struct instruction *instr,
5981                         struct instruction_data *data)
5982 {
5983         char *a = tokens[2], *b = tokens[3];
5984         struct field *fa, *fb;
5985         uint64_t b_val;
5986         uint32_t a_struct_id, b_struct_id;
5987
5988         CHECK(n_tokens == 4, EINVAL);
5989
5990         strcpy(data->jmp_label, tokens[1]);
5991
5992         fa = struct_field_parse(p, action, a, &a_struct_id);
5993         CHECK(fa, EINVAL);
5994         CHECK(!fa->var_size, EINVAL);
5995
5996         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5997         fb = struct_field_parse(p, action, b, &b_struct_id);
5998         if (fb) {
5999                 CHECK(!fb->var_size, EINVAL);
6000
6001                 instr->type = INSTR_JMP_NEQ;
6002                 if (a[0] != 'h' && b[0] == 'h')
6003                         instr->type = INSTR_JMP_NEQ_MH;
6004                 if (a[0] == 'h' && b[0] != 'h')
6005                         instr->type = INSTR_JMP_NEQ_HM;
6006                 if (a[0] == 'h' && b[0] == 'h')
6007                         instr->type = INSTR_JMP_NEQ_HH;
6008                 instr->jmp.ip = NULL; /* Resolved later. */
6009
6010                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6011                 instr->jmp.a.n_bits = fa->n_bits;
6012                 instr->jmp.a.offset = fa->offset / 8;
6013                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6014                 instr->jmp.b.n_bits = fb->n_bits;
6015                 instr->jmp.b.offset = fb->offset / 8;
6016                 return 0;
6017         }
6018
6019         /* JMP_NEQ_I. */
6020         b_val = strtoull(b, &b, 0);
6021         CHECK(!b[0], EINVAL);
6022
6023         if (a[0] == 'h')
6024                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6025
6026         instr->type = INSTR_JMP_NEQ_I;
6027         instr->jmp.ip = NULL; /* Resolved later. */
6028         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6029         instr->jmp.a.n_bits = fa->n_bits;
6030         instr->jmp.a.offset = fa->offset / 8;
6031         instr->jmp.b_val = b_val;
6032         return 0;
6033 }
6034
6035 static int
6036 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6037                        struct action *action,
6038                        char **tokens,
6039                        int n_tokens,
6040                        struct instruction *instr,
6041                        struct instruction_data *data)
6042 {
6043         char *a = tokens[2], *b = tokens[3];
6044         struct field *fa, *fb;
6045         uint64_t b_val;
6046         uint32_t a_struct_id, b_struct_id;
6047
6048         CHECK(n_tokens == 4, EINVAL);
6049
6050         strcpy(data->jmp_label, tokens[1]);
6051
6052         fa = struct_field_parse(p, action, a, &a_struct_id);
6053         CHECK(fa, EINVAL);
6054         CHECK(!fa->var_size, EINVAL);
6055
6056         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6057         fb = struct_field_parse(p, action, b, &b_struct_id);
6058         if (fb) {
6059                 CHECK(!fb->var_size, EINVAL);
6060
6061                 instr->type = INSTR_JMP_LT;
6062                 if (a[0] == 'h' && b[0] != 'h')
6063                         instr->type = INSTR_JMP_LT_HM;
6064                 if (a[0] != 'h' && b[0] == 'h')
6065                         instr->type = INSTR_JMP_LT_MH;
6066                 if (a[0] == 'h' && b[0] == 'h')
6067                         instr->type = INSTR_JMP_LT_HH;
6068                 instr->jmp.ip = NULL; /* Resolved later. */
6069
6070                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6071                 instr->jmp.a.n_bits = fa->n_bits;
6072                 instr->jmp.a.offset = fa->offset / 8;
6073                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6074                 instr->jmp.b.n_bits = fb->n_bits;
6075                 instr->jmp.b.offset = fb->offset / 8;
6076                 return 0;
6077         }
6078
6079         /* JMP_LT_MI, JMP_LT_HI. */
6080         b_val = strtoull(b, &b, 0);
6081         CHECK(!b[0], EINVAL);
6082
6083         instr->type = INSTR_JMP_LT_MI;
6084         if (a[0] == 'h')
6085                 instr->type = INSTR_JMP_LT_HI;
6086         instr->jmp.ip = NULL; /* Resolved later. */
6087
6088         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6089         instr->jmp.a.n_bits = fa->n_bits;
6090         instr->jmp.a.offset = fa->offset / 8;
6091         instr->jmp.b_val = b_val;
6092         return 0;
6093 }
6094
6095 static int
6096 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6097                        struct action *action,
6098                        char **tokens,
6099                        int n_tokens,
6100                        struct instruction *instr,
6101                        struct instruction_data *data)
6102 {
6103         char *a = tokens[2], *b = tokens[3];
6104         struct field *fa, *fb;
6105         uint64_t b_val;
6106         uint32_t a_struct_id, b_struct_id;
6107
6108         CHECK(n_tokens == 4, EINVAL);
6109
6110         strcpy(data->jmp_label, tokens[1]);
6111
6112         fa = struct_field_parse(p, action, a, &a_struct_id);
6113         CHECK(fa, EINVAL);
6114         CHECK(!fa->var_size, EINVAL);
6115
6116         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6117         fb = struct_field_parse(p, action, b, &b_struct_id);
6118         if (fb) {
6119                 CHECK(!fb->var_size, EINVAL);
6120
6121                 instr->type = INSTR_JMP_GT;
6122                 if (a[0] == 'h' && b[0] != 'h')
6123                         instr->type = INSTR_JMP_GT_HM;
6124                 if (a[0] != 'h' && b[0] == 'h')
6125                         instr->type = INSTR_JMP_GT_MH;
6126                 if (a[0] == 'h' && b[0] == 'h')
6127                         instr->type = INSTR_JMP_GT_HH;
6128                 instr->jmp.ip = NULL; /* Resolved later. */
6129
6130                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6131                 instr->jmp.a.n_bits = fa->n_bits;
6132                 instr->jmp.a.offset = fa->offset / 8;
6133                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6134                 instr->jmp.b.n_bits = fb->n_bits;
6135                 instr->jmp.b.offset = fb->offset / 8;
6136                 return 0;
6137         }
6138
6139         /* JMP_GT_MI, JMP_GT_HI. */
6140         b_val = strtoull(b, &b, 0);
6141         CHECK(!b[0], EINVAL);
6142
6143         instr->type = INSTR_JMP_GT_MI;
6144         if (a[0] == 'h')
6145                 instr->type = INSTR_JMP_GT_HI;
6146         instr->jmp.ip = NULL; /* Resolved later. */
6147
6148         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6149         instr->jmp.a.n_bits = fa->n_bits;
6150         instr->jmp.a.offset = fa->offset / 8;
6151         instr->jmp.b_val = b_val;
6152         return 0;
6153 }
6154
6155 static inline void
6156 instr_jmp_exec(struct rte_swx_pipeline *p)
6157 {
6158         struct thread *t = &p->threads[p->thread_id];
6159         struct instruction *ip = t->ip;
6160
6161         TRACE("[Thread %2u] jmp\n", p->thread_id);
6162
6163         thread_ip_set(t, ip->jmp.ip);
6164 }
6165
6166 static inline void
6167 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6168 {
6169         struct thread *t = &p->threads[p->thread_id];
6170         struct instruction *ip = t->ip;
6171         uint32_t header_id = ip->jmp.header_id;
6172
6173         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6174
6175         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6176 }
6177
6178 static inline void
6179 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6180 {
6181         struct thread *t = &p->threads[p->thread_id];
6182         struct instruction *ip = t->ip;
6183         uint32_t header_id = ip->jmp.header_id;
6184
6185         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6186
6187         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6188 }
6189
6190 static inline void
6191 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6192 {
6193         struct thread *t = &p->threads[p->thread_id];
6194         struct instruction *ip = t->ip;
6195         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6196
6197         TRACE("[Thread %2u] jmph\n", p->thread_id);
6198
6199         t->ip = ip_next[t->hit];
6200 }
6201
6202 static inline void
6203 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6204 {
6205         struct thread *t = &p->threads[p->thread_id];
6206         struct instruction *ip = t->ip;
6207         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6208
6209         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6210
6211         t->ip = ip_next[t->hit];
6212 }
6213
6214 static inline void
6215 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6216 {
6217         struct thread *t = &p->threads[p->thread_id];
6218         struct instruction *ip = t->ip;
6219
6220         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6221
6222         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6223 }
6224
6225 static inline void
6226 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6227 {
6228         struct thread *t = &p->threads[p->thread_id];
6229         struct instruction *ip = t->ip;
6230
6231         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6232
6233         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6234 }
6235
6236 static inline void
6237 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6238 {
6239         struct thread *t = &p->threads[p->thread_id];
6240         struct instruction *ip = t->ip;
6241
6242         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6243
6244         JMP_CMP(t, ip, ==);
6245 }
6246
6247 static inline void
6248 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6249 {
6250         struct thread *t = &p->threads[p->thread_id];
6251         struct instruction *ip = t->ip;
6252
6253         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6254
6255         JMP_CMP_MH(t, ip, ==);
6256 }
6257
6258 static inline void
6259 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6260 {
6261         struct thread *t = &p->threads[p->thread_id];
6262         struct instruction *ip = t->ip;
6263
6264         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6265
6266         JMP_CMP_HM(t, ip, ==);
6267 }
6268
6269 static inline void
6270 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6271 {
6272         struct thread *t = &p->threads[p->thread_id];
6273         struct instruction *ip = t->ip;
6274
6275         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6276
6277         JMP_CMP_HH_FAST(t, ip, ==);
6278 }
6279
6280 static inline void
6281 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6282 {
6283         struct thread *t = &p->threads[p->thread_id];
6284         struct instruction *ip = t->ip;
6285
6286         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6287
6288         JMP_CMP_I(t, ip, ==);
6289 }
6290
6291 static inline void
6292 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6293 {
6294         struct thread *t = &p->threads[p->thread_id];
6295         struct instruction *ip = t->ip;
6296
6297         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6298
6299         JMP_CMP(t, ip, !=);
6300 }
6301
6302 static inline void
6303 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6304 {
6305         struct thread *t = &p->threads[p->thread_id];
6306         struct instruction *ip = t->ip;
6307
6308         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6309
6310         JMP_CMP_MH(t, ip, !=);
6311 }
6312
6313 static inline void
6314 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6315 {
6316         struct thread *t = &p->threads[p->thread_id];
6317         struct instruction *ip = t->ip;
6318
6319         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6320
6321         JMP_CMP_HM(t, ip, !=);
6322 }
6323
6324 static inline void
6325 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6326 {
6327         struct thread *t = &p->threads[p->thread_id];
6328         struct instruction *ip = t->ip;
6329
6330         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6331
6332         JMP_CMP_HH_FAST(t, ip, !=);
6333 }
6334
6335 static inline void
6336 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6337 {
6338         struct thread *t = &p->threads[p->thread_id];
6339         struct instruction *ip = t->ip;
6340
6341         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6342
6343         JMP_CMP_I(t, ip, !=);
6344 }
6345
6346 static inline void
6347 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6348 {
6349         struct thread *t = &p->threads[p->thread_id];
6350         struct instruction *ip = t->ip;
6351
6352         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6353
6354         JMP_CMP(t, ip, <);
6355 }
6356
6357 static inline void
6358 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6359 {
6360         struct thread *t = &p->threads[p->thread_id];
6361         struct instruction *ip = t->ip;
6362
6363         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6364
6365         JMP_CMP_MH(t, ip, <);
6366 }
6367
6368 static inline void
6369 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6370 {
6371         struct thread *t = &p->threads[p->thread_id];
6372         struct instruction *ip = t->ip;
6373
6374         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6375
6376         JMP_CMP_HM(t, ip, <);
6377 }
6378
6379 static inline void
6380 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6381 {
6382         struct thread *t = &p->threads[p->thread_id];
6383         struct instruction *ip = t->ip;
6384
6385         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6386
6387         JMP_CMP_HH(t, ip, <);
6388 }
6389
6390 static inline void
6391 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6392 {
6393         struct thread *t = &p->threads[p->thread_id];
6394         struct instruction *ip = t->ip;
6395
6396         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6397
6398         JMP_CMP_MI(t, ip, <);
6399 }
6400
6401 static inline void
6402 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6403 {
6404         struct thread *t = &p->threads[p->thread_id];
6405         struct instruction *ip = t->ip;
6406
6407         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6408
6409         JMP_CMP_HI(t, ip, <);
6410 }
6411
6412 static inline void
6413 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6414 {
6415         struct thread *t = &p->threads[p->thread_id];
6416         struct instruction *ip = t->ip;
6417
6418         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6419
6420         JMP_CMP(t, ip, >);
6421 }
6422
6423 static inline void
6424 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6425 {
6426         struct thread *t = &p->threads[p->thread_id];
6427         struct instruction *ip = t->ip;
6428
6429         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6430
6431         JMP_CMP_MH(t, ip, >);
6432 }
6433
6434 static inline void
6435 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6436 {
6437         struct thread *t = &p->threads[p->thread_id];
6438         struct instruction *ip = t->ip;
6439
6440         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6441
6442         JMP_CMP_HM(t, ip, >);
6443 }
6444
6445 static inline void
6446 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6447 {
6448         struct thread *t = &p->threads[p->thread_id];
6449         struct instruction *ip = t->ip;
6450
6451         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6452
6453         JMP_CMP_HH(t, ip, >);
6454 }
6455
6456 static inline void
6457 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6458 {
6459         struct thread *t = &p->threads[p->thread_id];
6460         struct instruction *ip = t->ip;
6461
6462         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6463
6464         JMP_CMP_MI(t, ip, >);
6465 }
6466
6467 static inline void
6468 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6469 {
6470         struct thread *t = &p->threads[p->thread_id];
6471         struct instruction *ip = t->ip;
6472
6473         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6474
6475         JMP_CMP_HI(t, ip, >);
6476 }
6477
6478 /*
6479  * return.
6480  */
6481 static int
6482 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6483                        struct action *action,
6484                        char **tokens __rte_unused,
6485                        int n_tokens,
6486                        struct instruction *instr,
6487                        struct instruction_data *data __rte_unused)
6488 {
6489         CHECK(action, EINVAL);
6490         CHECK(n_tokens == 1, EINVAL);
6491
6492         instr->type = INSTR_RETURN;
6493         return 0;
6494 }
6495
6496 static inline void
6497 instr_return_exec(struct rte_swx_pipeline *p)
6498 {
6499         struct thread *t = &p->threads[p->thread_id];
6500
6501         TRACE("[Thread %2u] return\n", p->thread_id);
6502
6503         t->ip = t->ret;
6504 }
6505
6506 static int
6507 instr_translate(struct rte_swx_pipeline *p,
6508                 struct action *action,
6509                 char *string,
6510                 struct instruction *instr,
6511                 struct instruction_data *data)
6512 {
6513         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6514         int n_tokens = 0, tpos = 0;
6515
6516         /* Parse the instruction string into tokens. */
6517         for ( ; ; ) {
6518                 char *token;
6519
6520                 token = strtok_r(string, " \t\v", &string);
6521                 if (!token)
6522                         break;
6523
6524                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6525                 CHECK_NAME(token, EINVAL);
6526
6527                 tokens[n_tokens] = token;
6528                 n_tokens++;
6529         }
6530
6531         CHECK(n_tokens, EINVAL);
6532
6533         /* Handle the optional instruction label. */
6534         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6535                 strcpy(data->label, tokens[0]);
6536
6537                 tpos += 2;
6538                 CHECK(n_tokens - tpos, EINVAL);
6539         }
6540
6541         /* Identify the instruction type. */
6542         if (!strcmp(tokens[tpos], "rx"))
6543                 return instr_rx_translate(p,
6544                                           action,
6545                                           &tokens[tpos],
6546                                           n_tokens - tpos,
6547                                           instr,
6548                                           data);
6549
6550         if (!strcmp(tokens[tpos], "tx"))
6551                 return instr_tx_translate(p,
6552                                           action,
6553                                           &tokens[tpos],
6554                                           n_tokens - tpos,
6555                                           instr,
6556                                           data);
6557
6558         if (!strcmp(tokens[tpos], "drop"))
6559                 return instr_drop_translate(p,
6560                                             action,
6561                                             &tokens[tpos],
6562                                             n_tokens - tpos,
6563                                             instr,
6564                                             data);
6565
6566         if (!strcmp(tokens[tpos], "extract"))
6567                 return instr_hdr_extract_translate(p,
6568                                                    action,
6569                                                    &tokens[tpos],
6570                                                    n_tokens - tpos,
6571                                                    instr,
6572                                                    data);
6573
6574         if (!strcmp(tokens[tpos], "lookahead"))
6575                 return instr_hdr_lookahead_translate(p,
6576                                                      action,
6577                                                      &tokens[tpos],
6578                                                      n_tokens - tpos,
6579                                                      instr,
6580                                                      data);
6581
6582         if (!strcmp(tokens[tpos], "emit"))
6583                 return instr_hdr_emit_translate(p,
6584                                                 action,
6585                                                 &tokens[tpos],
6586                                                 n_tokens - tpos,
6587                                                 instr,
6588                                                 data);
6589
6590         if (!strcmp(tokens[tpos], "validate"))
6591                 return instr_hdr_validate_translate(p,
6592                                                     action,
6593                                                     &tokens[tpos],
6594                                                     n_tokens - tpos,
6595                                                     instr,
6596                                                     data);
6597
6598         if (!strcmp(tokens[tpos], "invalidate"))
6599                 return instr_hdr_invalidate_translate(p,
6600                                                       action,
6601                                                       &tokens[tpos],
6602                                                       n_tokens - tpos,
6603                                                       instr,
6604                                                       data);
6605
6606         if (!strcmp(tokens[tpos], "mov"))
6607                 return instr_mov_translate(p,
6608                                            action,
6609                                            &tokens[tpos],
6610                                            n_tokens - tpos,
6611                                            instr,
6612                                            data);
6613
6614         if (!strcmp(tokens[tpos], "add"))
6615                 return instr_alu_add_translate(p,
6616                                                action,
6617                                                &tokens[tpos],
6618                                                n_tokens - tpos,
6619                                                instr,
6620                                                data);
6621
6622         if (!strcmp(tokens[tpos], "sub"))
6623                 return instr_alu_sub_translate(p,
6624                                                action,
6625                                                &tokens[tpos],
6626                                                n_tokens - tpos,
6627                                                instr,
6628                                                data);
6629
6630         if (!strcmp(tokens[tpos], "ckadd"))
6631                 return instr_alu_ckadd_translate(p,
6632                                                  action,
6633                                                  &tokens[tpos],
6634                                                  n_tokens - tpos,
6635                                                  instr,
6636                                                  data);
6637
6638         if (!strcmp(tokens[tpos], "cksub"))
6639                 return instr_alu_cksub_translate(p,
6640                                                  action,
6641                                                  &tokens[tpos],
6642                                                  n_tokens - tpos,
6643                                                  instr,
6644                                                  data);
6645
6646         if (!strcmp(tokens[tpos], "and"))
6647                 return instr_alu_and_translate(p,
6648                                                action,
6649                                                &tokens[tpos],
6650                                                n_tokens - tpos,
6651                                                instr,
6652                                                data);
6653
6654         if (!strcmp(tokens[tpos], "or"))
6655                 return instr_alu_or_translate(p,
6656                                               action,
6657                                               &tokens[tpos],
6658                                               n_tokens - tpos,
6659                                               instr,
6660                                               data);
6661
6662         if (!strcmp(tokens[tpos], "xor"))
6663                 return instr_alu_xor_translate(p,
6664                                                action,
6665                                                &tokens[tpos],
6666                                                n_tokens - tpos,
6667                                                instr,
6668                                                data);
6669
6670         if (!strcmp(tokens[tpos], "shl"))
6671                 return instr_alu_shl_translate(p,
6672                                                action,
6673                                                &tokens[tpos],
6674                                                n_tokens - tpos,
6675                                                instr,
6676                                                data);
6677
6678         if (!strcmp(tokens[tpos], "shr"))
6679                 return instr_alu_shr_translate(p,
6680                                                action,
6681                                                &tokens[tpos],
6682                                                n_tokens - tpos,
6683                                                instr,
6684                                                data);
6685
6686         if (!strcmp(tokens[tpos], "regprefetch"))
6687                 return instr_regprefetch_translate(p,
6688                                                    action,
6689                                                    &tokens[tpos],
6690                                                    n_tokens - tpos,
6691                                                    instr,
6692                                                    data);
6693
6694         if (!strcmp(tokens[tpos], "regrd"))
6695                 return instr_regrd_translate(p,
6696                                              action,
6697                                              &tokens[tpos],
6698                                              n_tokens - tpos,
6699                                              instr,
6700                                              data);
6701
6702         if (!strcmp(tokens[tpos], "regwr"))
6703                 return instr_regwr_translate(p,
6704                                              action,
6705                                              &tokens[tpos],
6706                                              n_tokens - tpos,
6707                                              instr,
6708                                              data);
6709
6710         if (!strcmp(tokens[tpos], "regadd"))
6711                 return instr_regadd_translate(p,
6712                                               action,
6713                                               &tokens[tpos],
6714                                               n_tokens - tpos,
6715                                               instr,
6716                                               data);
6717
6718         if (!strcmp(tokens[tpos], "metprefetch"))
6719                 return instr_metprefetch_translate(p,
6720                                                    action,
6721                                                    &tokens[tpos],
6722                                                    n_tokens - tpos,
6723                                                    instr,
6724                                                    data);
6725
6726         if (!strcmp(tokens[tpos], "meter"))
6727                 return instr_meter_translate(p,
6728                                              action,
6729                                              &tokens[tpos],
6730                                              n_tokens - tpos,
6731                                              instr,
6732                                              data);
6733
6734         if (!strcmp(tokens[tpos], "table"))
6735                 return instr_table_translate(p,
6736                                              action,
6737                                              &tokens[tpos],
6738                                              n_tokens - tpos,
6739                                              instr,
6740                                              data);
6741
6742         if (!strcmp(tokens[tpos], "learn"))
6743                 return instr_learn_translate(p,
6744                                              action,
6745                                              &tokens[tpos],
6746                                              n_tokens - tpos,
6747                                              instr,
6748                                              data);
6749
6750         if (!strcmp(tokens[tpos], "forget"))
6751                 return instr_forget_translate(p,
6752                                               action,
6753                                               &tokens[tpos],
6754                                               n_tokens - tpos,
6755                                               instr,
6756                                               data);
6757
6758         if (!strcmp(tokens[tpos], "extern"))
6759                 return instr_extern_translate(p,
6760                                               action,
6761                                               &tokens[tpos],
6762                                               n_tokens - tpos,
6763                                               instr,
6764                                               data);
6765
6766         if (!strcmp(tokens[tpos], "jmp"))
6767                 return instr_jmp_translate(p,
6768                                            action,
6769                                            &tokens[tpos],
6770                                            n_tokens - tpos,
6771                                            instr,
6772                                            data);
6773
6774         if (!strcmp(tokens[tpos], "jmpv"))
6775                 return instr_jmp_valid_translate(p,
6776                                                  action,
6777                                                  &tokens[tpos],
6778                                                  n_tokens - tpos,
6779                                                  instr,
6780                                                  data);
6781
6782         if (!strcmp(tokens[tpos], "jmpnv"))
6783                 return instr_jmp_invalid_translate(p,
6784                                                    action,
6785                                                    &tokens[tpos],
6786                                                    n_tokens - tpos,
6787                                                    instr,
6788                                                    data);
6789
6790         if (!strcmp(tokens[tpos], "jmph"))
6791                 return instr_jmp_hit_translate(p,
6792                                                action,
6793                                                &tokens[tpos],
6794                                                n_tokens - tpos,
6795                                                instr,
6796                                                data);
6797
6798         if (!strcmp(tokens[tpos], "jmpnh"))
6799                 return instr_jmp_miss_translate(p,
6800                                                 action,
6801                                                 &tokens[tpos],
6802                                                 n_tokens - tpos,
6803                                                 instr,
6804                                                 data);
6805
6806         if (!strcmp(tokens[tpos], "jmpa"))
6807                 return instr_jmp_action_hit_translate(p,
6808                                                       action,
6809                                                       &tokens[tpos],
6810                                                       n_tokens - tpos,
6811                                                       instr,
6812                                                       data);
6813
6814         if (!strcmp(tokens[tpos], "jmpna"))
6815                 return instr_jmp_action_miss_translate(p,
6816                                                        action,
6817                                                        &tokens[tpos],
6818                                                        n_tokens - tpos,
6819                                                        instr,
6820                                                        data);
6821
6822         if (!strcmp(tokens[tpos], "jmpeq"))
6823                 return instr_jmp_eq_translate(p,
6824                                               action,
6825                                               &tokens[tpos],
6826                                               n_tokens - tpos,
6827                                               instr,
6828                                               data);
6829
6830         if (!strcmp(tokens[tpos], "jmpneq"))
6831                 return instr_jmp_neq_translate(p,
6832                                                action,
6833                                                &tokens[tpos],
6834                                                n_tokens - tpos,
6835                                                instr,
6836                                                data);
6837
6838         if (!strcmp(tokens[tpos], "jmplt"))
6839                 return instr_jmp_lt_translate(p,
6840                                               action,
6841                                               &tokens[tpos],
6842                                               n_tokens - tpos,
6843                                               instr,
6844                                               data);
6845
6846         if (!strcmp(tokens[tpos], "jmpgt"))
6847                 return instr_jmp_gt_translate(p,
6848                                               action,
6849                                               &tokens[tpos],
6850                                               n_tokens - tpos,
6851                                               instr,
6852                                               data);
6853
6854         if (!strcmp(tokens[tpos], "return"))
6855                 return instr_return_translate(p,
6856                                               action,
6857                                               &tokens[tpos],
6858                                               n_tokens - tpos,
6859                                               instr,
6860                                               data);
6861
6862         CHECK(0, EINVAL);
6863 }
6864
6865 static struct instruction_data *
6866 label_find(struct instruction_data *data, uint32_t n, const char *label)
6867 {
6868         uint32_t i;
6869
6870         for (i = 0; i < n; i++)
6871                 if (!strcmp(label, data[i].label))
6872                         return &data[i];
6873
6874         return NULL;
6875 }
6876
6877 static uint32_t
6878 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6879 {
6880         uint32_t count = 0, i;
6881
6882         if (!label[0])
6883                 return 0;
6884
6885         for (i = 0; i < n; i++)
6886                 if (!strcmp(label, data[i].jmp_label))
6887                         count++;
6888
6889         return count;
6890 }
6891
6892 static int
6893 instr_label_check(struct instruction_data *instruction_data,
6894                   uint32_t n_instructions)
6895 {
6896         uint32_t i;
6897
6898         /* Check that all instruction labels are unique. */
6899         for (i = 0; i < n_instructions; i++) {
6900                 struct instruction_data *data = &instruction_data[i];
6901                 char *label = data->label;
6902                 uint32_t j;
6903
6904                 if (!label[0])
6905                         continue;
6906
6907                 for (j = i + 1; j < n_instructions; j++)
6908                         CHECK(strcmp(label, data[j].label), EINVAL);
6909         }
6910
6911         /* Get users for each instruction label. */
6912         for (i = 0; i < n_instructions; i++) {
6913                 struct instruction_data *data = &instruction_data[i];
6914                 char *label = data->label;
6915
6916                 data->n_users = label_is_used(instruction_data,
6917                                               n_instructions,
6918                                               label);
6919         }
6920
6921         return 0;
6922 }
6923
6924 static int
6925 instr_jmp_resolve(struct instruction *instructions,
6926                   struct instruction_data *instruction_data,
6927                   uint32_t n_instructions)
6928 {
6929         uint32_t i;
6930
6931         for (i = 0; i < n_instructions; i++) {
6932                 struct instruction *instr = &instructions[i];
6933                 struct instruction_data *data = &instruction_data[i];
6934                 struct instruction_data *found;
6935
6936                 if (!instruction_is_jmp(instr))
6937                         continue;
6938
6939                 found = label_find(instruction_data,
6940                                    n_instructions,
6941                                    data->jmp_label);
6942                 CHECK(found, EINVAL);
6943
6944                 instr->jmp.ip = &instructions[found - instruction_data];
6945         }
6946
6947         return 0;
6948 }
6949
6950 static int
6951 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6952              struct action *a,
6953              struct instruction *instr,
6954              struct instruction_data *data __rte_unused,
6955              uint32_t n_instructions)
6956 {
6957         if (!a) {
6958                 enum instruction_type type;
6959                 uint32_t i;
6960
6961                 /* Check that the first instruction is rx. */
6962                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6963
6964                 /* Check that there is at least one tx instruction. */
6965                 for (i = 0; i < n_instructions; i++) {
6966                         type = instr[i].type;
6967
6968                         if (instruction_is_tx(type))
6969                                 break;
6970                 }
6971                 CHECK(i < n_instructions, EINVAL);
6972
6973                 /* Check that the last instruction is either tx or unconditional
6974                  * jump.
6975                  */
6976                 type = instr[n_instructions - 1].type;
6977                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6978         }
6979
6980         if (a) {
6981                 enum instruction_type type;
6982                 uint32_t i;
6983
6984                 /* Check that there is at least one return or tx instruction. */
6985                 for (i = 0; i < n_instructions; i++) {
6986                         type = instr[i].type;
6987
6988                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6989                                 break;
6990                 }
6991                 CHECK(i < n_instructions, EINVAL);
6992         }
6993
6994         return 0;
6995 }
6996
6997 static uint32_t
6998 instr_compact(struct instruction *instructions,
6999               struct instruction_data *instruction_data,
7000               uint32_t n_instructions)
7001 {
7002         uint32_t i, pos = 0;
7003
7004         /* Eliminate the invalid instructions that have been optimized out. */
7005         for (i = 0; i < n_instructions; i++) {
7006                 struct instruction *instr = &instructions[i];
7007                 struct instruction_data *data = &instruction_data[i];
7008
7009                 if (data->invalid)
7010                         continue;
7011
7012                 if (i != pos) {
7013                         memcpy(&instructions[pos], instr, sizeof(*instr));
7014                         memcpy(&instruction_data[pos], data, sizeof(*data));
7015                 }
7016
7017                 pos++;
7018         }
7019
7020         return pos;
7021 }
7022
7023 static int
7024 instr_pattern_extract_many_search(struct instruction *instr,
7025                                   struct instruction_data *data,
7026                                   uint32_t n_instr,
7027                                   uint32_t *n_pattern_instr)
7028 {
7029         uint32_t i;
7030
7031         for (i = 0; i < n_instr; i++) {
7032                 if (data[i].invalid)
7033                         break;
7034
7035                 if (instr[i].type != INSTR_HDR_EXTRACT)
7036                         break;
7037
7038                 if (i == RTE_DIM(instr->io.hdr.header_id))
7039                         break;
7040
7041                 if (i && data[i].n_users)
7042                         break;
7043         }
7044
7045         if (i < 2)
7046                 return 0;
7047
7048         *n_pattern_instr = i;
7049         return 1;
7050 }
7051
7052 static void
7053 instr_pattern_extract_many_replace(struct instruction *instr,
7054                                    struct instruction_data *data,
7055                                    uint32_t n_instr)
7056 {
7057         uint32_t i;
7058
7059         for (i = 1; i < n_instr; i++) {
7060                 instr[0].type++;
7061                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7062                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7063                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7064
7065                 data[i].invalid = 1;
7066         }
7067 }
7068
7069 static uint32_t
7070 instr_pattern_extract_many_optimize(struct instruction *instructions,
7071                                     struct instruction_data *instruction_data,
7072                                     uint32_t n_instructions)
7073 {
7074         uint32_t i;
7075
7076         for (i = 0; i < n_instructions; ) {
7077                 struct instruction *instr = &instructions[i];
7078                 struct instruction_data *data = &instruction_data[i];
7079                 uint32_t n_instr = 0;
7080                 int detected;
7081
7082                 /* Extract many. */
7083                 detected = instr_pattern_extract_many_search(instr,
7084                                                              data,
7085                                                              n_instructions - i,
7086                                                              &n_instr);
7087                 if (detected) {
7088                         instr_pattern_extract_many_replace(instr,
7089                                                            data,
7090                                                            n_instr);
7091                         i += n_instr;
7092                         continue;
7093                 }
7094
7095                 /* No pattern starting at the current instruction. */
7096                 i++;
7097         }
7098
7099         /* Eliminate the invalid instructions that have been optimized out. */
7100         n_instructions = instr_compact(instructions,
7101                                        instruction_data,
7102                                        n_instructions);
7103
7104         return n_instructions;
7105 }
7106
7107 static int
7108 instr_pattern_emit_many_tx_search(struct instruction *instr,
7109                                   struct instruction_data *data,
7110                                   uint32_t n_instr,
7111                                   uint32_t *n_pattern_instr)
7112 {
7113         uint32_t i;
7114
7115         for (i = 0; i < n_instr; i++) {
7116                 if (data[i].invalid)
7117                         break;
7118
7119                 if (instr[i].type != INSTR_HDR_EMIT)
7120                         break;
7121
7122                 if (i == RTE_DIM(instr->io.hdr.header_id))
7123                         break;
7124
7125                 if (i && data[i].n_users)
7126                         break;
7127         }
7128
7129         if (!i)
7130                 return 0;
7131
7132         if (!instruction_is_tx(instr[i].type))
7133                 return 0;
7134
7135         if (data[i].n_users)
7136                 return 0;
7137
7138         i++;
7139
7140         *n_pattern_instr = i;
7141         return 1;
7142 }
7143
7144 static void
7145 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7146                                    struct instruction_data *data,
7147                                    uint32_t n_instr)
7148 {
7149         uint32_t i;
7150
7151         /* Any emit instruction in addition to the first one. */
7152         for (i = 1; i < n_instr - 1; i++) {
7153                 instr[0].type++;
7154                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7155                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7156                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7157
7158                 data[i].invalid = 1;
7159         }
7160
7161         /* The TX instruction is the last one in the pattern. */
7162         instr[0].type++;
7163         instr[0].io.io.offset = instr[i].io.io.offset;
7164         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7165         data[i].invalid = 1;
7166 }
7167
7168 static uint32_t
7169 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7170                                     struct instruction_data *instruction_data,
7171                                     uint32_t n_instructions)
7172 {
7173         uint32_t i;
7174
7175         for (i = 0; i < n_instructions; ) {
7176                 struct instruction *instr = &instructions[i];
7177                 struct instruction_data *data = &instruction_data[i];
7178                 uint32_t n_instr = 0;
7179                 int detected;
7180
7181                 /* Emit many + TX. */
7182                 detected = instr_pattern_emit_many_tx_search(instr,
7183                                                              data,
7184                                                              n_instructions - i,
7185                                                              &n_instr);
7186                 if (detected) {
7187                         instr_pattern_emit_many_tx_replace(instr,
7188                                                            data,
7189                                                            n_instr);
7190                         i += n_instr;
7191                         continue;
7192                 }
7193
7194                 /* No pattern starting at the current instruction. */
7195                 i++;
7196         }
7197
7198         /* Eliminate the invalid instructions that have been optimized out. */
7199         n_instructions = instr_compact(instructions,
7200                                        instruction_data,
7201                                        n_instructions);
7202
7203         return n_instructions;
7204 }
7205
7206 static uint32_t
7207 action_arg_src_mov_count(struct action *a,
7208                          uint32_t arg_id,
7209                          struct instruction *instructions,
7210                          struct instruction_data *instruction_data,
7211                          uint32_t n_instructions);
7212
7213 static int
7214 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7215                                       struct action *a,
7216                                       struct instruction *instr,
7217                                       struct instruction_data *data,
7218                                       uint32_t n_instr,
7219                                       struct instruction *instructions,
7220                                       struct instruction_data *instruction_data,
7221                                       uint32_t n_instructions,
7222                                       uint32_t *n_pattern_instr)
7223 {
7224         struct header *h;
7225         uint32_t src_field_id, i, j;
7226
7227         /* Prerequisites. */
7228         if (!a || !a->st)
7229                 return 0;
7230
7231         /* First instruction: MOV_HM. */
7232         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7233                 return 0;
7234
7235         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7236         if (!h || h->st->var_size)
7237                 return 0;
7238
7239         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7240                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7241                         break;
7242
7243         if (src_field_id == a->st->n_fields)
7244                 return 0;
7245
7246         if (instr[0].mov.dst.offset ||
7247             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7248             instr[0].mov.src.struct_id ||
7249             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7250             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7251                 return 0;
7252
7253         if ((n_instr < h->st->n_fields + 1) ||
7254              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7255                 return 0;
7256
7257         /* Subsequent instructions: MOV_HM. */
7258         for (i = 1; i < h->st->n_fields; i++)
7259                 if (data[i].invalid ||
7260                     data[i].n_users ||
7261                     (instr[i].type != INSTR_MOV_HM) ||
7262                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7263                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7264                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7265                     instr[i].mov.src.struct_id ||
7266                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7267                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7268                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7269                         return 0;
7270
7271         /* Last instruction: HDR_VALIDATE. */
7272         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7273             (instr[i].valid.header_id != h->id))
7274                 return 0;
7275
7276         /* Check that none of the action args that are used as source for this
7277          * DMA transfer are not used as source in any other mov instruction.
7278          */
7279         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7280                 uint32_t n_users;
7281
7282                 n_users = action_arg_src_mov_count(a,
7283                                                    j,
7284                                                    instructions,
7285                                                    instruction_data,
7286                                                    n_instructions);
7287                 if (n_users > 1)
7288                         return 0;
7289         }
7290
7291         *n_pattern_instr = 1 + i;
7292         return 1;
7293 }
7294
7295 static void
7296 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7297                                        struct action *a,
7298                                        struct instruction *instr,
7299                                        struct instruction_data *data,
7300                                        uint32_t n_instr)
7301 {
7302         struct header *h;
7303         uint32_t src_field_id, src_offset, i;
7304
7305         /* Read from the instructions before they are modified. */
7306         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7307         if (!h)
7308                 return;
7309
7310         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7311                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7312                         break;
7313
7314         if (src_field_id == a->st->n_fields)
7315                 return;
7316
7317         src_offset = instr[0].mov.src.offset;
7318
7319         /* Modify the instructions. */
7320         instr[0].type = INSTR_DMA_HT;
7321         instr[0].dma.dst.header_id[0] = h->id;
7322         instr[0].dma.dst.struct_id[0] = h->struct_id;
7323         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7324         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7325
7326         for (i = 1; i < n_instr; i++)
7327                 data[i].invalid = 1;
7328
7329         /* Update the endianness of the action arguments to header endianness. */
7330         for (i = 0; i < h->st->n_fields; i++)
7331                 a->args_endianness[src_field_id + i] = 1;
7332 }
7333
7334 static uint32_t
7335 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7336                                         struct action *a,
7337                                         struct instruction *instructions,
7338                                         struct instruction_data *instruction_data,
7339                                         uint32_t n_instructions)
7340 {
7341         uint32_t i;
7342
7343         if (!a || !a->st)
7344                 return n_instructions;
7345
7346         for (i = 0; i < n_instructions; ) {
7347                 struct instruction *instr = &instructions[i];
7348                 struct instruction_data *data = &instruction_data[i];
7349                 uint32_t n_instr = 0;
7350                 int detected;
7351
7352                 /* Mov all + validate. */
7353                 detected = instr_pattern_mov_all_validate_search(p,
7354                                                                  a,
7355                                                                  instr,
7356                                                                  data,
7357                                                                  n_instructions - i,
7358                                                                  instructions,
7359                                                                  instruction_data,
7360                                                                  n_instructions,
7361                                                                  &n_instr);
7362                 if (detected) {
7363                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7364                         i += n_instr;
7365                         continue;
7366                 }
7367
7368                 /* No pattern starting at the current instruction. */
7369                 i++;
7370         }
7371
7372         /* Eliminate the invalid instructions that have been optimized out. */
7373         n_instructions = instr_compact(instructions,
7374                                        instruction_data,
7375                                        n_instructions);
7376
7377         return n_instructions;
7378 }
7379
7380 static int
7381 instr_pattern_dma_many_search(struct instruction *instr,
7382                               struct instruction_data *data,
7383                               uint32_t n_instr,
7384                               uint32_t *n_pattern_instr)
7385 {
7386         uint32_t i;
7387
7388         for (i = 0; i < n_instr; i++) {
7389                 if (data[i].invalid)
7390                         break;
7391
7392                 if (instr[i].type != INSTR_DMA_HT)
7393                         break;
7394
7395                 if (i == RTE_DIM(instr->dma.dst.header_id))
7396                         break;
7397
7398                 if (i && data[i].n_users)
7399                         break;
7400         }
7401
7402         if (i < 2)
7403                 return 0;
7404
7405         *n_pattern_instr = i;
7406         return 1;
7407 }
7408
7409 static void
7410 instr_pattern_dma_many_replace(struct instruction *instr,
7411                                struct instruction_data *data,
7412                                uint32_t n_instr)
7413 {
7414         uint32_t i;
7415
7416         for (i = 1; i < n_instr; i++) {
7417                 instr[0].type++;
7418                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7419                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7420                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7421                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7422
7423                 data[i].invalid = 1;
7424         }
7425 }
7426
7427 static uint32_t
7428 instr_pattern_dma_many_optimize(struct instruction *instructions,
7429                struct instruction_data *instruction_data,
7430                uint32_t n_instructions)
7431 {
7432         uint32_t i;
7433
7434         for (i = 0; i < n_instructions; ) {
7435                 struct instruction *instr = &instructions[i];
7436                 struct instruction_data *data = &instruction_data[i];
7437                 uint32_t n_instr = 0;
7438                 int detected;
7439
7440                 /* DMA many. */
7441                 detected = instr_pattern_dma_many_search(instr,
7442                                                          data,
7443                                                          n_instructions - i,
7444                                                          &n_instr);
7445                 if (detected) {
7446                         instr_pattern_dma_many_replace(instr, data, n_instr);
7447                         i += n_instr;
7448                         continue;
7449                 }
7450
7451                 /* No pattern starting at the current instruction. */
7452                 i++;
7453         }
7454
7455         /* Eliminate the invalid instructions that have been optimized out. */
7456         n_instructions = instr_compact(instructions,
7457                                        instruction_data,
7458                                        n_instructions);
7459
7460         return n_instructions;
7461 }
7462
7463 static uint32_t
7464 instr_optimize(struct rte_swx_pipeline *p,
7465                struct action *a,
7466                struct instruction *instructions,
7467                struct instruction_data *instruction_data,
7468                uint32_t n_instructions)
7469 {
7470         /* Extract many. */
7471         n_instructions = instr_pattern_extract_many_optimize(instructions,
7472                                                              instruction_data,
7473                                                              n_instructions);
7474
7475         /* Emit many + TX. */
7476         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7477                                                              instruction_data,
7478                                                              n_instructions);
7479
7480         /* Mov all + validate. */
7481         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7482                                                                  a,
7483                                                                  instructions,
7484                                                                  instruction_data,
7485                                                                  n_instructions);
7486
7487         /* DMA many. */
7488         n_instructions = instr_pattern_dma_many_optimize(instructions,
7489                                                          instruction_data,
7490                                                          n_instructions);
7491
7492         return n_instructions;
7493 }
7494
7495 static int
7496 instruction_config(struct rte_swx_pipeline *p,
7497                    struct action *a,
7498                    const char **instructions,
7499                    uint32_t n_instructions)
7500 {
7501         struct instruction *instr = NULL;
7502         struct instruction_data *data = NULL;
7503         int err = 0;
7504         uint32_t i;
7505
7506         CHECK(n_instructions, EINVAL);
7507         CHECK(instructions, EINVAL);
7508         for (i = 0; i < n_instructions; i++)
7509                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7510
7511         /* Memory allocation. */
7512         instr = calloc(n_instructions, sizeof(struct instruction));
7513         if (!instr) {
7514                 err = -ENOMEM;
7515                 goto error;
7516         }
7517
7518         data = calloc(n_instructions, sizeof(struct instruction_data));
7519         if (!data) {
7520                 err = -ENOMEM;
7521                 goto error;
7522         }
7523
7524         for (i = 0; i < n_instructions; i++) {
7525                 char *string = strdup(instructions[i]);
7526                 if (!string) {
7527                         err = -ENOMEM;
7528                         goto error;
7529                 }
7530
7531                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7532                 if (err) {
7533                         free(string);
7534                         goto error;
7535                 }
7536
7537                 free(string);
7538         }
7539
7540         err = instr_label_check(data, n_instructions);
7541         if (err)
7542                 goto error;
7543
7544         err = instr_verify(p, a, instr, data, n_instructions);
7545         if (err)
7546                 goto error;
7547
7548         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7549
7550         err = instr_jmp_resolve(instr, data, n_instructions);
7551         if (err)
7552                 goto error;
7553
7554         if (a) {
7555                 a->instructions = instr;
7556                 a->n_instructions = n_instructions;
7557         } else {
7558                 p->instructions = instr;
7559                 p->n_instructions = n_instructions;
7560         }
7561
7562         free(data);
7563         return 0;
7564
7565 error:
7566         free(data);
7567         free(instr);
7568         return err;
7569 }
7570
7571 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7572
7573 static instr_exec_t instruction_table[] = {
7574         [INSTR_RX] = instr_rx_exec,
7575         [INSTR_TX] = instr_tx_exec,
7576         [INSTR_TX_I] = instr_tx_i_exec,
7577
7578         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7579         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7580         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7581         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7582         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7583         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7584         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7585         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7586         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7587         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7588
7589         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7590         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7591         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7592         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7593         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7594         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7595         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7596         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7597         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7598
7599         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7600         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7601
7602         [INSTR_MOV] = instr_mov_exec,
7603         [INSTR_MOV_MH] = instr_mov_mh_exec,
7604         [INSTR_MOV_HM] = instr_mov_hm_exec,
7605         [INSTR_MOV_HH] = instr_mov_hh_exec,
7606         [INSTR_MOV_I] = instr_mov_i_exec,
7607
7608         [INSTR_DMA_HT] = instr_dma_ht_exec,
7609         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7610         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7611         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7612         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7613         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7614         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7615         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7616
7617         [INSTR_ALU_ADD] = instr_alu_add_exec,
7618         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7619         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7620         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7621         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7622         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7623
7624         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7625         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7626         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7627         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7628         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7629         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7630
7631         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7632         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7633         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7634         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7635
7636         [INSTR_ALU_AND] = instr_alu_and_exec,
7637         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7638         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7639         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7640         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7641
7642         [INSTR_ALU_OR] = instr_alu_or_exec,
7643         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7644         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7645         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7646         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7647
7648         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7649         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7650         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7651         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7652         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7653
7654         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7655         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7656         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7657         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7658         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7659         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7660
7661         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7662         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7663         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7664         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7665         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7666         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7667
7668         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7669         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7670         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7671
7672         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7673         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7674         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7675         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7676         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7677         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7678
7679         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7680         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7681         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7682         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7683         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7684         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7685         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7686         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7687         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7688
7689         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7690         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7691         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7692         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7693         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7694         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7695         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7696         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7697         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7698
7699         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7700         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7701         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7702
7703         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7704         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7705         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7706         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7707         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7708         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7709         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7710         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7711         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7712         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7713         [INSTR_METER_IMM] = instr_meter_imm_exec,
7714         [INSTR_METER_IMI] = instr_meter_imi_exec,
7715
7716         [INSTR_TABLE] = instr_table_exec,
7717         [INSTR_SELECTOR] = instr_selector_exec,
7718         [INSTR_LEARNER] = instr_learner_exec,
7719         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7720         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7721         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7722         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7723
7724         [INSTR_JMP] = instr_jmp_exec,
7725         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7726         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7727         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7728         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7729         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7730         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7731
7732         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7733         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7734         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7735         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7736         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7737
7738         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7739         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7740         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7741         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7742         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7743
7744         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7745         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7746         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7747         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7748         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7749         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7750
7751         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7752         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7753         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7754         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7755         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7756         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7757
7758         [INSTR_RETURN] = instr_return_exec,
7759 };
7760
7761 static inline void
7762 instr_exec(struct rte_swx_pipeline *p)
7763 {
7764         struct thread *t = &p->threads[p->thread_id];
7765         struct instruction *ip = t->ip;
7766         instr_exec_t instr = instruction_table[ip->type];
7767
7768         instr(p);
7769 }
7770
7771 /*
7772  * Action.
7773  */
7774 static struct action *
7775 action_find(struct rte_swx_pipeline *p, const char *name)
7776 {
7777         struct action *elem;
7778
7779         if (!name)
7780                 return NULL;
7781
7782         TAILQ_FOREACH(elem, &p->actions, node)
7783                 if (strcmp(elem->name, name) == 0)
7784                         return elem;
7785
7786         return NULL;
7787 }
7788
7789 static struct action *
7790 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7791 {
7792         struct action *action = NULL;
7793
7794         TAILQ_FOREACH(action, &p->actions, node)
7795                 if (action->id == id)
7796                         return action;
7797
7798         return NULL;
7799 }
7800
7801 static struct field *
7802 action_field_find(struct action *a, const char *name)
7803 {
7804         return a->st ? struct_type_field_find(a->st, name) : NULL;
7805 }
7806
7807 static struct field *
7808 action_field_parse(struct action *action, const char *name)
7809 {
7810         if (name[0] != 't' || name[1] != '.')
7811                 return NULL;
7812
7813         return action_field_find(action, &name[2]);
7814 }
7815
7816 static int
7817 action_has_nbo_args(struct action *a)
7818 {
7819         uint32_t i;
7820
7821         /* Return if the action does not have any args. */
7822         if (!a->st)
7823                 return 0; /* FALSE */
7824
7825         for (i = 0; i < a->st->n_fields; i++)
7826                 if (a->args_endianness[i])
7827                         return 1; /* TRUE */
7828
7829         return 0; /* FALSE */
7830 }
7831
7832 static int
7833 action_does_learning(struct action *a)
7834 {
7835         uint32_t i;
7836
7837         for (i = 0; i < a->n_instructions; i++)
7838                 switch (a->instructions[i].type) {
7839                 case INSTR_LEARNER_LEARN:
7840                         return 1; /* TRUE */
7841
7842                 case INSTR_LEARNER_FORGET:
7843                         return 1; /* TRUE */
7844
7845                 default:
7846                         continue;
7847                 }
7848
7849         return 0; /* FALSE */
7850 }
7851
7852 int
7853 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7854                                const char *name,
7855                                const char *args_struct_type_name,
7856                                const char **instructions,
7857                                uint32_t n_instructions)
7858 {
7859         struct struct_type *args_struct_type = NULL;
7860         struct action *a;
7861         int err;
7862
7863         CHECK(p, EINVAL);
7864
7865         CHECK_NAME(name, EINVAL);
7866         CHECK(!action_find(p, name), EEXIST);
7867
7868         if (args_struct_type_name) {
7869                 CHECK_NAME(args_struct_type_name, EINVAL);
7870                 args_struct_type = struct_type_find(p, args_struct_type_name);
7871                 CHECK(args_struct_type, EINVAL);
7872                 CHECK(!args_struct_type->var_size, EINVAL);
7873         }
7874
7875         /* Node allocation. */
7876         a = calloc(1, sizeof(struct action));
7877         CHECK(a, ENOMEM);
7878         if (args_struct_type) {
7879                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7880                 if (!a->args_endianness) {
7881                         free(a);
7882                         CHECK(0, ENOMEM);
7883                 }
7884         }
7885
7886         /* Node initialization. */
7887         strcpy(a->name, name);
7888         a->st = args_struct_type;
7889         a->id = p->n_actions;
7890
7891         /* Instruction translation. */
7892         err = instruction_config(p, a, instructions, n_instructions);
7893         if (err) {
7894                 free(a->args_endianness);
7895                 free(a);
7896                 return err;
7897         }
7898
7899         /* Node add to tailq. */
7900         TAILQ_INSERT_TAIL(&p->actions, a, node);
7901         p->n_actions++;
7902
7903         return 0;
7904 }
7905
7906 static int
7907 action_build(struct rte_swx_pipeline *p)
7908 {
7909         struct action *action;
7910
7911         p->action_instructions = calloc(p->n_actions,
7912                                         sizeof(struct instruction *));
7913         CHECK(p->action_instructions, ENOMEM);
7914
7915         TAILQ_FOREACH(action, &p->actions, node)
7916                 p->action_instructions[action->id] = action->instructions;
7917
7918         return 0;
7919 }
7920
7921 static void
7922 action_build_free(struct rte_swx_pipeline *p)
7923 {
7924         free(p->action_instructions);
7925         p->action_instructions = NULL;
7926 }
7927
7928 static void
7929 action_free(struct rte_swx_pipeline *p)
7930 {
7931         action_build_free(p);
7932
7933         for ( ; ; ) {
7934                 struct action *action;
7935
7936                 action = TAILQ_FIRST(&p->actions);
7937                 if (!action)
7938                         break;
7939
7940                 TAILQ_REMOVE(&p->actions, action, node);
7941                 free(action->instructions);
7942                 free(action);
7943         }
7944 }
7945
7946 static uint32_t
7947 action_arg_src_mov_count(struct action *a,
7948                          uint32_t arg_id,
7949                          struct instruction *instructions,
7950                          struct instruction_data *instruction_data,
7951                          uint32_t n_instructions)
7952 {
7953         uint32_t offset, n_users = 0, i;
7954
7955         if (!a->st ||
7956             (arg_id >= a->st->n_fields) ||
7957             !instructions ||
7958             !instruction_data ||
7959             !n_instructions)
7960                 return 0;
7961
7962         offset = a->st->fields[arg_id].offset / 8;
7963
7964         for (i = 0; i < n_instructions; i++) {
7965                 struct instruction *instr = &instructions[i];
7966                 struct instruction_data *data = &instruction_data[i];
7967
7968                 if (data->invalid ||
7969                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7970                     instr->mov.src.struct_id ||
7971                     (instr->mov.src.offset != offset))
7972                         continue;
7973
7974                 n_users++;
7975         }
7976
7977         return n_users;
7978 }
7979
7980 /*
7981  * Table.
7982  */
7983 static struct table_type *
7984 table_type_find(struct rte_swx_pipeline *p, const char *name)
7985 {
7986         struct table_type *elem;
7987
7988         TAILQ_FOREACH(elem, &p->table_types, node)
7989                 if (strcmp(elem->name, name) == 0)
7990                         return elem;
7991
7992         return NULL;
7993 }
7994
7995 static struct table_type *
7996 table_type_resolve(struct rte_swx_pipeline *p,
7997                    const char *recommended_type_name,
7998                    enum rte_swx_table_match_type match_type)
7999 {
8000         struct table_type *elem;
8001
8002         /* Only consider the recommended type if the match type is correct. */
8003         if (recommended_type_name)
8004                 TAILQ_FOREACH(elem, &p->table_types, node)
8005                         if (!strcmp(elem->name, recommended_type_name) &&
8006                             (elem->match_type == match_type))
8007                                 return elem;
8008
8009         /* Ignore the recommended type and get the first element with this match
8010          * type.
8011          */
8012         TAILQ_FOREACH(elem, &p->table_types, node)
8013                 if (elem->match_type == match_type)
8014                         return elem;
8015
8016         return NULL;
8017 }
8018
8019 static struct table *
8020 table_find(struct rte_swx_pipeline *p, const char *name)
8021 {
8022         struct table *elem;
8023
8024         TAILQ_FOREACH(elem, &p->tables, node)
8025                 if (strcmp(elem->name, name) == 0)
8026                         return elem;
8027
8028         return NULL;
8029 }
8030
8031 static struct table *
8032 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8033 {
8034         struct table *table = NULL;
8035
8036         TAILQ_FOREACH(table, &p->tables, node)
8037                 if (table->id == id)
8038                         return table;
8039
8040         return NULL;
8041 }
8042
8043 int
8044 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8045                                      const char *name,
8046                                      enum rte_swx_table_match_type match_type,
8047                                      struct rte_swx_table_ops *ops)
8048 {
8049         struct table_type *elem;
8050
8051         CHECK(p, EINVAL);
8052
8053         CHECK_NAME(name, EINVAL);
8054         CHECK(!table_type_find(p, name), EEXIST);
8055
8056         CHECK(ops, EINVAL);
8057         CHECK(ops->create, EINVAL);
8058         CHECK(ops->lkp, EINVAL);
8059         CHECK(ops->free, EINVAL);
8060
8061         /* Node allocation. */
8062         elem = calloc(1, sizeof(struct table_type));
8063         CHECK(elem, ENOMEM);
8064
8065         /* Node initialization. */
8066         strcpy(elem->name, name);
8067         elem->match_type = match_type;
8068         memcpy(&elem->ops, ops, sizeof(*ops));
8069
8070         /* Node add to tailq. */
8071         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8072
8073         return 0;
8074 }
8075
8076 static int
8077 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8078                          uint32_t n_fields,
8079                          enum rte_swx_table_match_type *match_type)
8080 {
8081         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
8082
8083         for (i = 0; i < n_fields; i++) {
8084                 struct rte_swx_match_field_params  *f = &fields[i];
8085
8086                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
8087                         n_fields_em++;
8088
8089                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
8090                         n_fields_lpm++;
8091         }
8092
8093         if ((n_fields_lpm > 1) ||
8094             (n_fields_lpm && (n_fields_em != n_fields - 1)))
8095                 return -EINVAL;
8096
8097         *match_type = (n_fields_em == n_fields) ?
8098                        RTE_SWX_TABLE_MATCH_EXACT :
8099                        RTE_SWX_TABLE_MATCH_WILDCARD;
8100
8101         return 0;
8102 }
8103
8104 static int
8105 table_match_fields_check(struct rte_swx_pipeline *p,
8106                          struct rte_swx_pipeline_table_params *params,
8107                          struct header **header)
8108 {
8109         struct header *h0 = NULL;
8110         struct field *hf, *mf;
8111         uint32_t *offset = NULL, i;
8112         int status = 0;
8113
8114         /* Return if no match fields. */
8115         if (!params->n_fields) {
8116                 if (params->fields) {
8117                         status = -EINVAL;
8118                         goto end;
8119                 }
8120
8121                 if (header)
8122                         *header = NULL;
8123
8124                 return 0;
8125         }
8126
8127         /* Memory allocation. */
8128         offset = calloc(params->n_fields, sizeof(uint32_t));
8129         if (!offset) {
8130                 status = -ENOMEM;
8131                 goto end;
8132         }
8133
8134         /* Check that all the match fields belong to either the same header or
8135          * to the meta-data.
8136          */
8137         hf = header_field_parse(p, params->fields[0].name, &h0);
8138         mf = metadata_field_parse(p, params->fields[0].name);
8139         if ((!hf && !mf) || (hf && hf->var_size)) {
8140                 status = -EINVAL;
8141                 goto end;
8142         }
8143
8144         offset[0] = h0 ? hf->offset : mf->offset;
8145
8146         for (i = 1; i < params->n_fields; i++)
8147                 if (h0) {
8148                         struct header *h;
8149
8150                         hf = header_field_parse(p, params->fields[i].name, &h);
8151                         if (!hf || (h->id != h0->id) || hf->var_size) {
8152                                 status = -EINVAL;
8153                                 goto end;
8154                         }
8155
8156                         offset[i] = hf->offset;
8157                 } else {
8158                         mf = metadata_field_parse(p, params->fields[i].name);
8159                         if (!mf) {
8160                                 status = -EINVAL;
8161                                 goto end;
8162                         }
8163
8164                         offset[i] = mf->offset;
8165                 }
8166
8167         /* Check that there are no duplicated match fields. */
8168         for (i = 0; i < params->n_fields; i++) {
8169                 uint32_t j;
8170
8171                 for (j = 0; j < i; j++)
8172                         if (offset[j] == offset[i]) {
8173                                 status = -EINVAL;
8174                                 goto end;
8175                         }
8176         }
8177
8178         /* Return. */
8179         if (header)
8180                 *header = h0;
8181
8182 end:
8183         free(offset);
8184         return status;
8185 }
8186
8187 int
8188 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8189                               const char *name,
8190                               struct rte_swx_pipeline_table_params *params,
8191                               const char *recommended_table_type_name,
8192                               const char *args,
8193                               uint32_t size)
8194 {
8195         struct table_type *type;
8196         struct table *t;
8197         struct action *default_action;
8198         struct header *header = NULL;
8199         uint32_t action_data_size_max = 0, i;
8200         int status = 0;
8201
8202         CHECK(p, EINVAL);
8203
8204         CHECK_NAME(name, EINVAL);
8205         CHECK(!table_find(p, name), EEXIST);
8206         CHECK(!selector_find(p, name), EEXIST);
8207         CHECK(!learner_find(p, name), EEXIST);
8208
8209         CHECK(params, EINVAL);
8210
8211         /* Match checks. */
8212         status = table_match_fields_check(p, params, &header);
8213         if (status)
8214                 return status;
8215
8216         /* Action checks. */
8217         CHECK(params->n_actions, EINVAL);
8218         CHECK(params->action_names, EINVAL);
8219         for (i = 0; i < params->n_actions; i++) {
8220                 const char *action_name = params->action_names[i];
8221                 struct action *a;
8222                 uint32_t action_data_size;
8223
8224                 CHECK_NAME(action_name, EINVAL);
8225
8226                 a = action_find(p, action_name);
8227                 CHECK(a, EINVAL);
8228                 CHECK(!action_does_learning(a), EINVAL);
8229
8230                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8231                 if (action_data_size > action_data_size_max)
8232                         action_data_size_max = action_data_size;
8233         }
8234
8235         CHECK_NAME(params->default_action_name, EINVAL);
8236         for (i = 0; i < p->n_actions; i++)
8237                 if (!strcmp(params->action_names[i],
8238                             params->default_action_name))
8239                         break;
8240         CHECK(i < params->n_actions, EINVAL);
8241         default_action = action_find(p, params->default_action_name);
8242         CHECK((default_action->st && params->default_action_data) ||
8243               !params->default_action_data, EINVAL);
8244
8245         /* Table type checks. */
8246         if (recommended_table_type_name)
8247                 CHECK_NAME(recommended_table_type_name, EINVAL);
8248
8249         if (params->n_fields) {
8250                 enum rte_swx_table_match_type match_type;
8251
8252                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8253                 if (status)
8254                         return status;
8255
8256                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8257                 CHECK(type, EINVAL);
8258         } else {
8259                 type = NULL;
8260         }
8261
8262         /* Memory allocation. */
8263         t = calloc(1, sizeof(struct table));
8264         CHECK(t, ENOMEM);
8265
8266         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8267         if (!t->fields) {
8268                 free(t);
8269                 CHECK(0, ENOMEM);
8270         }
8271
8272         t->actions = calloc(params->n_actions, sizeof(struct action *));
8273         if (!t->actions) {
8274                 free(t->fields);
8275                 free(t);
8276                 CHECK(0, ENOMEM);
8277         }
8278
8279         if (action_data_size_max) {
8280                 t->default_action_data = calloc(1, action_data_size_max);
8281                 if (!t->default_action_data) {
8282                         free(t->actions);
8283                         free(t->fields);
8284                         free(t);
8285                         CHECK(0, ENOMEM);
8286                 }
8287         }
8288
8289         /* Node initialization. */
8290         strcpy(t->name, name);
8291         if (args && args[0])
8292                 strcpy(t->args, args);
8293         t->type = type;
8294
8295         for (i = 0; i < params->n_fields; i++) {
8296                 struct rte_swx_match_field_params *field = &params->fields[i];
8297                 struct match_field *f = &t->fields[i];
8298
8299                 f->match_type = field->match_type;
8300                 f->field = header ?
8301                         header_field_parse(p, field->name, NULL) :
8302                         metadata_field_parse(p, field->name);
8303         }
8304         t->n_fields = params->n_fields;
8305         t->header = header;
8306
8307         for (i = 0; i < params->n_actions; i++)
8308                 t->actions[i] = action_find(p, params->action_names[i]);
8309         t->default_action = default_action;
8310         if (default_action->st)
8311                 memcpy(t->default_action_data,
8312                        params->default_action_data,
8313                        default_action->st->n_bits / 8);
8314         t->n_actions = params->n_actions;
8315         t->default_action_is_const = params->default_action_is_const;
8316         t->action_data_size_max = action_data_size_max;
8317
8318         t->size = size;
8319         t->id = p->n_tables;
8320
8321         /* Node add to tailq. */
8322         TAILQ_INSERT_TAIL(&p->tables, t, node);
8323         p->n_tables++;
8324
8325         return 0;
8326 }
8327
8328 static struct rte_swx_table_params *
8329 table_params_get(struct table *table)
8330 {
8331         struct rte_swx_table_params *params;
8332         struct field *first, *last;
8333         uint8_t *key_mask;
8334         uint32_t key_size, key_offset, action_data_size, i;
8335
8336         /* Memory allocation. */
8337         params = calloc(1, sizeof(struct rte_swx_table_params));
8338         if (!params)
8339                 return NULL;
8340
8341         /* Find first (smallest offset) and last (biggest offset) match fields. */
8342         first = table->fields[0].field;
8343         last = table->fields[0].field;
8344
8345         for (i = 0; i < table->n_fields; i++) {
8346                 struct field *f = table->fields[i].field;
8347
8348                 if (f->offset < first->offset)
8349                         first = f;
8350
8351                 if (f->offset > last->offset)
8352                         last = f;
8353         }
8354
8355         /* Key offset and size. */
8356         key_offset = first->offset / 8;
8357         key_size = (last->offset + last->n_bits - first->offset) / 8;
8358
8359         /* Memory allocation. */
8360         key_mask = calloc(1, key_size);
8361         if (!key_mask) {
8362                 free(params);
8363                 return NULL;
8364         }
8365
8366         /* Key mask. */
8367         for (i = 0; i < table->n_fields; i++) {
8368                 struct field *f = table->fields[i].field;
8369                 uint32_t start = (f->offset - first->offset) / 8;
8370                 size_t size = f->n_bits / 8;
8371
8372                 memset(&key_mask[start], 0xFF, size);
8373         }
8374
8375         /* Action data size. */
8376         action_data_size = 0;
8377         for (i = 0; i < table->n_actions; i++) {
8378                 struct action *action = table->actions[i];
8379                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8380
8381                 if (ads > action_data_size)
8382                         action_data_size = ads;
8383         }
8384
8385         /* Fill in. */
8386         params->match_type = table->type->match_type;
8387         params->key_size = key_size;
8388         params->key_offset = key_offset;
8389         params->key_mask0 = key_mask;
8390         params->action_data_size = action_data_size;
8391         params->n_keys_max = table->size;
8392
8393         return params;
8394 }
8395
8396 static void
8397 table_params_free(struct rte_swx_table_params *params)
8398 {
8399         if (!params)
8400                 return;
8401
8402         free(params->key_mask0);
8403         free(params);
8404 }
8405
8406 static int
8407 table_stub_lkp(void *table __rte_unused,
8408                void *mailbox __rte_unused,
8409                uint8_t **key __rte_unused,
8410                uint64_t *action_id __rte_unused,
8411                uint8_t **action_data __rte_unused,
8412                int *hit)
8413 {
8414         *hit = 0;
8415         return 1; /* DONE. */
8416 }
8417
8418 static int
8419 table_build(struct rte_swx_pipeline *p)
8420 {
8421         uint32_t i;
8422
8423         /* Per pipeline: table statistics. */
8424         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8425         CHECK(p->table_stats, ENOMEM);
8426
8427         for (i = 0; i < p->n_tables; i++) {
8428                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8429                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8430         }
8431
8432         /* Per thread: table runt-time. */
8433         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8434                 struct thread *t = &p->threads[i];
8435                 struct table *table;
8436
8437                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8438                 CHECK(t->tables, ENOMEM);
8439
8440                 TAILQ_FOREACH(table, &p->tables, node) {
8441                         struct table_runtime *r = &t->tables[table->id];
8442
8443                         if (table->type) {
8444                                 uint64_t size;
8445
8446                                 size = table->type->ops.mailbox_size_get();
8447
8448                                 /* r->func. */
8449                                 r->func = table->type->ops.lkp;
8450
8451                                 /* r->mailbox. */
8452                                 if (size) {
8453                                         r->mailbox = calloc(1, size);
8454                                         CHECK(r->mailbox, ENOMEM);
8455                                 }
8456
8457                                 /* r->key. */
8458                                 r->key = table->header ?
8459                                         &t->structs[table->header->struct_id] :
8460                                         &t->structs[p->metadata_struct_id];
8461                         } else {
8462                                 r->func = table_stub_lkp;
8463                         }
8464                 }
8465         }
8466
8467         return 0;
8468 }
8469
8470 static void
8471 table_build_free(struct rte_swx_pipeline *p)
8472 {
8473         uint32_t i;
8474
8475         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8476                 struct thread *t = &p->threads[i];
8477                 uint32_t j;
8478
8479                 if (!t->tables)
8480                         continue;
8481
8482                 for (j = 0; j < p->n_tables; j++) {
8483                         struct table_runtime *r = &t->tables[j];
8484
8485                         free(r->mailbox);
8486                 }
8487
8488                 free(t->tables);
8489                 t->tables = NULL;
8490         }
8491
8492         if (p->table_stats) {
8493                 for (i = 0; i < p->n_tables; i++)
8494                         free(p->table_stats[i].n_pkts_action);
8495
8496                 free(p->table_stats);
8497         }
8498 }
8499
8500 static void
8501 table_free(struct rte_swx_pipeline *p)
8502 {
8503         table_build_free(p);
8504
8505         /* Tables. */
8506         for ( ; ; ) {
8507                 struct table *elem;
8508
8509                 elem = TAILQ_FIRST(&p->tables);
8510                 if (!elem)
8511                         break;
8512
8513                 TAILQ_REMOVE(&p->tables, elem, node);
8514                 free(elem->fields);
8515                 free(elem->actions);
8516                 free(elem->default_action_data);
8517                 free(elem);
8518         }
8519
8520         /* Table types. */
8521         for ( ; ; ) {
8522                 struct table_type *elem;
8523
8524                 elem = TAILQ_FIRST(&p->table_types);
8525                 if (!elem)
8526                         break;
8527
8528                 TAILQ_REMOVE(&p->table_types, elem, node);
8529                 free(elem);
8530         }
8531 }
8532
8533 /*
8534  * Selector.
8535  */
8536 static struct selector *
8537 selector_find(struct rte_swx_pipeline *p, const char *name)
8538 {
8539         struct selector *s;
8540
8541         TAILQ_FOREACH(s, &p->selectors, node)
8542                 if (strcmp(s->name, name) == 0)
8543                         return s;
8544
8545         return NULL;
8546 }
8547
8548 static struct selector *
8549 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8550 {
8551         struct selector *s = NULL;
8552
8553         TAILQ_FOREACH(s, &p->selectors, node)
8554                 if (s->id == id)
8555                         return s;
8556
8557         return NULL;
8558 }
8559
8560 static int
8561 selector_fields_check(struct rte_swx_pipeline *p,
8562                       struct rte_swx_pipeline_selector_params *params,
8563                       struct header **header)
8564 {
8565         struct header *h0 = NULL;
8566         struct field *hf, *mf;
8567         uint32_t i;
8568
8569         /* Return if no selector fields. */
8570         if (!params->n_selector_fields || !params->selector_field_names)
8571                 return -EINVAL;
8572
8573         /* Check that all the selector fields either belong to the same header
8574          * or are all meta-data fields.
8575          */
8576         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8577         mf = metadata_field_parse(p, params->selector_field_names[0]);
8578         if (!hf && !mf)
8579                 return -EINVAL;
8580
8581         for (i = 1; i < params->n_selector_fields; i++)
8582                 if (h0) {
8583                         struct header *h;
8584
8585                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8586                         if (!hf || (h->id != h0->id))
8587                                 return -EINVAL;
8588                 } else {
8589                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8590                         if (!mf)
8591                                 return -EINVAL;
8592                 }
8593
8594         /* Check that there are no duplicated match fields. */
8595         for (i = 0; i < params->n_selector_fields; i++) {
8596                 const char *field_name = params->selector_field_names[i];
8597                 uint32_t j;
8598
8599                 for (j = i + 1; j < params->n_selector_fields; j++)
8600                         if (!strcmp(params->selector_field_names[j], field_name))
8601                                 return -EINVAL;
8602         }
8603
8604         /* Return. */
8605         if (header)
8606                 *header = h0;
8607
8608         return 0;
8609 }
8610
8611 int
8612 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8613                                  const char *name,
8614                                  struct rte_swx_pipeline_selector_params *params)
8615 {
8616         struct selector *s;
8617         struct header *selector_header = NULL;
8618         struct field *group_id_field, *member_id_field;
8619         uint32_t i;
8620         int status = 0;
8621
8622         CHECK(p, EINVAL);
8623
8624         CHECK_NAME(name, EINVAL);
8625         CHECK(!table_find(p, name), EEXIST);
8626         CHECK(!selector_find(p, name), EEXIST);
8627         CHECK(!learner_find(p, name), EEXIST);
8628
8629         CHECK(params, EINVAL);
8630
8631         CHECK_NAME(params->group_id_field_name, EINVAL);
8632         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8633         CHECK(group_id_field, EINVAL);
8634
8635         for (i = 0; i < params->n_selector_fields; i++) {
8636                 const char *field_name = params->selector_field_names[i];
8637
8638                 CHECK_NAME(field_name, EINVAL);
8639         }
8640         status = selector_fields_check(p, params, &selector_header);
8641         if (status)
8642                 return status;
8643
8644         CHECK_NAME(params->member_id_field_name, EINVAL);
8645         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8646         CHECK(member_id_field, EINVAL);
8647
8648         CHECK(params->n_groups_max, EINVAL);
8649
8650         CHECK(params->n_members_per_group_max, EINVAL);
8651
8652         /* Memory allocation. */
8653         s = calloc(1, sizeof(struct selector));
8654         if (!s) {
8655                 status = -ENOMEM;
8656                 goto error;
8657         }
8658
8659         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8660         if (!s->selector_fields) {
8661                 status = -ENOMEM;
8662                 goto error;
8663         }
8664
8665         /* Node initialization. */
8666         strcpy(s->name, name);
8667
8668         s->group_id_field = group_id_field;
8669
8670         for (i = 0; i < params->n_selector_fields; i++) {
8671                 const char *field_name = params->selector_field_names[i];
8672
8673                 s->selector_fields[i] = selector_header ?
8674                         header_field_parse(p, field_name, NULL) :
8675                         metadata_field_parse(p, field_name);
8676         }
8677
8678         s->n_selector_fields = params->n_selector_fields;
8679
8680         s->selector_header = selector_header;
8681
8682         s->member_id_field = member_id_field;
8683
8684         s->n_groups_max = params->n_groups_max;
8685
8686         s->n_members_per_group_max = params->n_members_per_group_max;
8687
8688         s->id = p->n_selectors;
8689
8690         /* Node add to tailq. */
8691         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8692         p->n_selectors++;
8693
8694         return 0;
8695
8696 error:
8697         if (!s)
8698                 return status;
8699
8700         free(s->selector_fields);
8701
8702         free(s);
8703
8704         return status;
8705 }
8706
8707 static void
8708 selector_params_free(struct rte_swx_table_selector_params *params)
8709 {
8710         if (!params)
8711                 return;
8712
8713         free(params->selector_mask);
8714
8715         free(params);
8716 }
8717
8718 static struct rte_swx_table_selector_params *
8719 selector_table_params_get(struct selector *s)
8720 {
8721         struct rte_swx_table_selector_params *params = NULL;
8722         struct field *first, *last;
8723         uint32_t i;
8724
8725         /* Memory allocation. */
8726         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8727         if (!params)
8728                 goto error;
8729
8730         /* Group ID. */
8731         params->group_id_offset = s->group_id_field->offset / 8;
8732
8733         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8734         first = s->selector_fields[0];
8735         last = s->selector_fields[0];
8736
8737         for (i = 0; i < s->n_selector_fields; i++) {
8738                 struct field *f = s->selector_fields[i];
8739
8740                 if (f->offset < first->offset)
8741                         first = f;
8742
8743                 if (f->offset > last->offset)
8744                         last = f;
8745         }
8746
8747         /* Selector offset and size. */
8748         params->selector_offset = first->offset / 8;
8749         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8750
8751         /* Memory allocation. */
8752         params->selector_mask = calloc(1, params->selector_size);
8753         if (!params->selector_mask)
8754                 goto error;
8755
8756         /* Selector mask. */
8757         for (i = 0; i < s->n_selector_fields; i++) {
8758                 struct field *f = s->selector_fields[i];
8759                 uint32_t start = (f->offset - first->offset) / 8;
8760                 size_t size = f->n_bits / 8;
8761
8762                 memset(&params->selector_mask[start], 0xFF, size);
8763         }
8764
8765         /* Member ID. */
8766         params->member_id_offset = s->member_id_field->offset / 8;
8767
8768         /* Maximum number of groups. */
8769         params->n_groups_max = s->n_groups_max;
8770
8771         /* Maximum number of members per group. */
8772         params->n_members_per_group_max = s->n_members_per_group_max;
8773
8774         return params;
8775
8776 error:
8777         selector_params_free(params);
8778         return NULL;
8779 }
8780
8781 static void
8782 selector_build_free(struct rte_swx_pipeline *p)
8783 {
8784         uint32_t i;
8785
8786         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8787                 struct thread *t = &p->threads[i];
8788                 uint32_t j;
8789
8790                 if (!t->selectors)
8791                         continue;
8792
8793                 for (j = 0; j < p->n_selectors; j++) {
8794                         struct selector_runtime *r = &t->selectors[j];
8795
8796                         free(r->mailbox);
8797                 }
8798
8799                 free(t->selectors);
8800                 t->selectors = NULL;
8801         }
8802
8803         free(p->selector_stats);
8804         p->selector_stats = NULL;
8805 }
8806
8807 static int
8808 selector_build(struct rte_swx_pipeline *p)
8809 {
8810         uint32_t i;
8811         int status = 0;
8812
8813         /* Per pipeline: selector statistics. */
8814         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8815         if (!p->selector_stats) {
8816                 status = -ENOMEM;
8817                 goto error;
8818         }
8819
8820         /* Per thread: selector run-time. */
8821         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8822                 struct thread *t = &p->threads[i];
8823                 struct selector *s;
8824
8825                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8826                 if (!t->selectors) {
8827                         status = -ENOMEM;
8828                         goto error;
8829                 }
8830
8831                 TAILQ_FOREACH(s, &p->selectors, node) {
8832                         struct selector_runtime *r = &t->selectors[s->id];
8833                         uint64_t size;
8834
8835                         /* r->mailbox. */
8836                         size = rte_swx_table_selector_mailbox_size_get();
8837                         if (size) {
8838                                 r->mailbox = calloc(1, size);
8839                                 if (!r->mailbox) {
8840                                         status = -ENOMEM;
8841                                         goto error;
8842                                 }
8843                         }
8844
8845                         /* r->group_id_buffer. */
8846                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8847
8848                         /* r->selector_buffer. */
8849                         r->selector_buffer = s->selector_header ?
8850                                 &t->structs[s->selector_header->struct_id] :
8851                                 &t->structs[p->metadata_struct_id];
8852
8853                         /* r->member_id_buffer. */
8854                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8855                 }
8856         }
8857
8858         return 0;
8859
8860 error:
8861         selector_build_free(p);
8862         return status;
8863 }
8864
8865 static void
8866 selector_free(struct rte_swx_pipeline *p)
8867 {
8868         selector_build_free(p);
8869
8870         /* Selector tables. */
8871         for ( ; ; ) {
8872                 struct selector *elem;
8873
8874                 elem = TAILQ_FIRST(&p->selectors);
8875                 if (!elem)
8876                         break;
8877
8878                 TAILQ_REMOVE(&p->selectors, elem, node);
8879                 free(elem->selector_fields);
8880                 free(elem);
8881         }
8882 }
8883
8884 /*
8885  * Learner table.
8886  */
8887 static struct learner *
8888 learner_find(struct rte_swx_pipeline *p, const char *name)
8889 {
8890         struct learner *l;
8891
8892         TAILQ_FOREACH(l, &p->learners, node)
8893                 if (!strcmp(l->name, name))
8894                         return l;
8895
8896         return NULL;
8897 }
8898
8899 static struct learner *
8900 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8901 {
8902         struct learner *l = NULL;
8903
8904         TAILQ_FOREACH(l, &p->learners, node)
8905                 if (l->id == id)
8906                         return l;
8907
8908         return NULL;
8909 }
8910
8911 static int
8912 learner_match_fields_check(struct rte_swx_pipeline *p,
8913                            struct rte_swx_pipeline_learner_params *params,
8914                            struct header **header)
8915 {
8916         struct header *h0 = NULL;
8917         struct field *hf, *mf;
8918         uint32_t i;
8919
8920         /* Return if no match fields. */
8921         if (!params->n_fields || !params->field_names)
8922                 return -EINVAL;
8923
8924         /* Check that all the match fields either belong to the same header
8925          * or are all meta-data fields.
8926          */
8927         hf = header_field_parse(p, params->field_names[0], &h0);
8928         mf = metadata_field_parse(p, params->field_names[0]);
8929         if (!hf && !mf)
8930                 return -EINVAL;
8931
8932         for (i = 1; i < params->n_fields; i++)
8933                 if (h0) {
8934                         struct header *h;
8935
8936                         hf = header_field_parse(p, params->field_names[i], &h);
8937                         if (!hf || (h->id != h0->id))
8938                                 return -EINVAL;
8939                 } else {
8940                         mf = metadata_field_parse(p, params->field_names[i]);
8941                         if (!mf)
8942                                 return -EINVAL;
8943                 }
8944
8945         /* Check that there are no duplicated match fields. */
8946         for (i = 0; i < params->n_fields; i++) {
8947                 const char *field_name = params->field_names[i];
8948                 uint32_t j;
8949
8950                 for (j = i + 1; j < params->n_fields; j++)
8951                         if (!strcmp(params->field_names[j], field_name))
8952                                 return -EINVAL;
8953         }
8954
8955         /* Return. */
8956         if (header)
8957                 *header = h0;
8958
8959         return 0;
8960 }
8961
8962 static int
8963 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8964 {
8965         struct struct_type *mst = p->metadata_st, *ast = a->st;
8966         struct field *mf, *af;
8967         uint32_t mf_pos, i;
8968
8969         if (!ast) {
8970                 if (mf_name)
8971                         return -EINVAL;
8972
8973                 return 0;
8974         }
8975
8976         /* Check that mf_name is the name of a valid meta-data field. */
8977         CHECK_NAME(mf_name, EINVAL);
8978         mf = metadata_field_parse(p, mf_name);
8979         CHECK(mf, EINVAL);
8980
8981         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8982          * all the action arguments.
8983          */
8984         mf_pos = mf - mst->fields;
8985         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8986
8987         /* Check that the size of each of the identified meta-data fields matches exactly the size
8988          * of the corresponding action argument.
8989          */
8990         for (i = 0; i < ast->n_fields; i++) {
8991                 mf = &mst->fields[mf_pos + i];
8992                 af = &ast->fields[i];
8993
8994                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8995         }
8996
8997         return 0;
8998 }
8999
9000 static int
9001 learner_action_learning_check(struct rte_swx_pipeline *p,
9002                               struct action *action,
9003                               const char **action_names,
9004                               uint32_t n_actions)
9005 {
9006         uint32_t i;
9007
9008         /* For each "learn" instruction of the current action, check that the learned action (i.e.
9009          * the action passed as argument to the "learn" instruction) is also enabled for the
9010          * current learner table.
9011          */
9012         for (i = 0; i < action->n_instructions; i++) {
9013                 struct instruction *instr = &action->instructions[i];
9014                 uint32_t found = 0, j;
9015
9016                 if (instr->type != INSTR_LEARNER_LEARN)
9017                         continue;
9018
9019                 for (j = 0; j < n_actions; j++) {
9020                         struct action *a;
9021
9022                         a = action_find(p, action_names[j]);
9023                         if (!a)
9024                                 return -EINVAL;
9025
9026                         if (a->id == instr->learn.action_id)
9027                                 found = 1;
9028                 }
9029
9030                 if (!found)
9031                         return -EINVAL;
9032         }
9033
9034         return 0;
9035 }
9036
9037 int
9038 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
9039                               const char *name,
9040                               struct rte_swx_pipeline_learner_params *params,
9041                               uint32_t size,
9042                               uint32_t timeout)
9043 {
9044         struct learner *l = NULL;
9045         struct action *default_action;
9046         struct header *header = NULL;
9047         uint32_t action_data_size_max = 0, i;
9048         int status = 0;
9049
9050         CHECK(p, EINVAL);
9051
9052         CHECK_NAME(name, EINVAL);
9053         CHECK(!table_find(p, name), EEXIST);
9054         CHECK(!selector_find(p, name), EEXIST);
9055         CHECK(!learner_find(p, name), EEXIST);
9056
9057         CHECK(params, EINVAL);
9058
9059         /* Match checks. */
9060         status = learner_match_fields_check(p, params, &header);
9061         if (status)
9062                 return status;
9063
9064         /* Action checks. */
9065         CHECK(params->n_actions, EINVAL);
9066
9067         CHECK(params->action_names, EINVAL);
9068         for (i = 0; i < params->n_actions; i++) {
9069                 const char *action_name = params->action_names[i];
9070                 const char *action_field_name = params->action_field_names[i];
9071                 struct action *a;
9072                 uint32_t action_data_size;
9073
9074                 CHECK_NAME(action_name, EINVAL);
9075
9076                 a = action_find(p, action_name);
9077                 CHECK(a, EINVAL);
9078
9079                 status = learner_action_args_check(p, a, action_field_name);
9080                 if (status)
9081                         return status;
9082
9083                 status = learner_action_learning_check(p,
9084                                                        a,
9085                                                        params->action_names,
9086                                                        params->n_actions);
9087                 if (status)
9088                         return status;
9089
9090                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9091                 if (action_data_size > action_data_size_max)
9092                         action_data_size_max = action_data_size;
9093         }
9094
9095         CHECK_NAME(params->default_action_name, EINVAL);
9096         for (i = 0; i < p->n_actions; i++)
9097                 if (!strcmp(params->action_names[i],
9098                             params->default_action_name))
9099                         break;
9100         CHECK(i < params->n_actions, EINVAL);
9101
9102         default_action = action_find(p, params->default_action_name);
9103         CHECK((default_action->st && params->default_action_data) ||
9104               !params->default_action_data, EINVAL);
9105
9106         /* Any other checks. */
9107         CHECK(size, EINVAL);
9108         CHECK(timeout, EINVAL);
9109
9110         /* Memory allocation. */
9111         l = calloc(1, sizeof(struct learner));
9112         if (!l)
9113                 goto nomem;
9114
9115         l->fields = calloc(params->n_fields, sizeof(struct field *));
9116         if (!l->fields)
9117                 goto nomem;
9118
9119         l->actions = calloc(params->n_actions, sizeof(struct action *));
9120         if (!l->actions)
9121                 goto nomem;
9122
9123         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9124         if (!l->action_arg)
9125                 goto nomem;
9126
9127         if (action_data_size_max) {
9128                 l->default_action_data = calloc(1, action_data_size_max);
9129                 if (!l->default_action_data)
9130                         goto nomem;
9131         }
9132
9133         /* Node initialization. */
9134         strcpy(l->name, name);
9135
9136         for (i = 0; i < params->n_fields; i++) {
9137                 const char *field_name = params->field_names[i];
9138
9139                 l->fields[i] = header ?
9140                         header_field_parse(p, field_name, NULL) :
9141                         metadata_field_parse(p, field_name);
9142         }
9143
9144         l->n_fields = params->n_fields;
9145
9146         l->header = header;
9147
9148         for (i = 0; i < params->n_actions; i++) {
9149                 const char *mf_name = params->action_field_names[i];
9150
9151                 l->actions[i] = action_find(p, params->action_names[i]);
9152
9153                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9154         }
9155
9156         l->default_action = default_action;
9157
9158         if (default_action->st)
9159                 memcpy(l->default_action_data,
9160                        params->default_action_data,
9161                        default_action->st->n_bits / 8);
9162
9163         l->n_actions = params->n_actions;
9164
9165         l->default_action_is_const = params->default_action_is_const;
9166
9167         l->action_data_size_max = action_data_size_max;
9168
9169         l->size = size;
9170
9171         l->timeout = timeout;
9172
9173         l->id = p->n_learners;
9174
9175         /* Node add to tailq. */
9176         TAILQ_INSERT_TAIL(&p->learners, l, node);
9177         p->n_learners++;
9178
9179         return 0;
9180
9181 nomem:
9182         if (!l)
9183                 return -ENOMEM;
9184
9185         free(l->action_arg);
9186         free(l->actions);
9187         free(l->fields);
9188         free(l);
9189
9190         return -ENOMEM;
9191 }
9192
9193 static void
9194 learner_params_free(struct rte_swx_table_learner_params *params)
9195 {
9196         if (!params)
9197                 return;
9198
9199         free(params->key_mask0);
9200
9201         free(params);
9202 }
9203
9204 static struct rte_swx_table_learner_params *
9205 learner_params_get(struct learner *l)
9206 {
9207         struct rte_swx_table_learner_params *params = NULL;
9208         struct field *first, *last;
9209         uint32_t i;
9210
9211         /* Memory allocation. */
9212         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9213         if (!params)
9214                 goto error;
9215
9216         /* Find first (smallest offset) and last (biggest offset) match fields. */
9217         first = l->fields[0];
9218         last = l->fields[0];
9219
9220         for (i = 0; i < l->n_fields; i++) {
9221                 struct field *f = l->fields[i];
9222
9223                 if (f->offset < first->offset)
9224                         first = f;
9225
9226                 if (f->offset > last->offset)
9227                         last = f;
9228         }
9229
9230         /* Key offset and size. */
9231         params->key_offset = first->offset / 8;
9232         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9233
9234         /* Memory allocation. */
9235         params->key_mask0 = calloc(1, params->key_size);
9236         if (!params->key_mask0)
9237                 goto error;
9238
9239         /* Key mask. */
9240         for (i = 0; i < l->n_fields; i++) {
9241                 struct field *f = l->fields[i];
9242                 uint32_t start = (f->offset - first->offset) / 8;
9243                 size_t size = f->n_bits / 8;
9244
9245                 memset(&params->key_mask0[start], 0xFF, size);
9246         }
9247
9248         /* Action data size. */
9249         params->action_data_size = l->action_data_size_max;
9250
9251         /* Maximum number of keys. */
9252         params->n_keys_max = l->size;
9253
9254         /* Timeout. */
9255         params->key_timeout = l->timeout;
9256
9257         return params;
9258
9259 error:
9260         learner_params_free(params);
9261         return NULL;
9262 }
9263
9264 static void
9265 learner_build_free(struct rte_swx_pipeline *p)
9266 {
9267         uint32_t i;
9268
9269         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9270                 struct thread *t = &p->threads[i];
9271                 uint32_t j;
9272
9273                 if (!t->learners)
9274                         continue;
9275
9276                 for (j = 0; j < p->n_learners; j++) {
9277                         struct learner_runtime *r = &t->learners[j];
9278
9279                         free(r->mailbox);
9280                         free(r->action_data);
9281                 }
9282
9283                 free(t->learners);
9284                 t->learners = NULL;
9285         }
9286
9287         if (p->learner_stats) {
9288                 for (i = 0; i < p->n_learners; i++)
9289                         free(p->learner_stats[i].n_pkts_action);
9290
9291                 free(p->learner_stats);
9292         }
9293 }
9294
9295 static int
9296 learner_build(struct rte_swx_pipeline *p)
9297 {
9298         uint32_t i;
9299         int status = 0;
9300
9301         /* Per pipeline: learner statistics. */
9302         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9303         CHECK(p->learner_stats, ENOMEM);
9304
9305         for (i = 0; i < p->n_learners; i++) {
9306                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9307                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9308         }
9309
9310         /* Per thread: learner run-time. */
9311         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9312                 struct thread *t = &p->threads[i];
9313                 struct learner *l;
9314
9315                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9316                 if (!t->learners) {
9317                         status = -ENOMEM;
9318                         goto error;
9319                 }
9320
9321                 TAILQ_FOREACH(l, &p->learners, node) {
9322                         struct learner_runtime *r = &t->learners[l->id];
9323                         uint64_t size;
9324                         uint32_t j;
9325
9326                         /* r->mailbox. */
9327                         size = rte_swx_table_learner_mailbox_size_get();
9328                         if (size) {
9329                                 r->mailbox = calloc(1, size);
9330                                 if (!r->mailbox) {
9331                                         status = -ENOMEM;
9332                                         goto error;
9333                                 }
9334                         }
9335
9336                         /* r->key. */
9337                         r->key = l->header ?
9338                                 &t->structs[l->header->struct_id] :
9339                                 &t->structs[p->metadata_struct_id];
9340
9341                         /* r->action_data. */
9342                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9343                         if (!r->action_data) {
9344                                 status = -ENOMEM;
9345                                 goto error;
9346                         }
9347
9348                         for (j = 0; j < l->n_actions; j++) {
9349                                 struct action *a = l->actions[j];
9350                                 struct field *mf = l->action_arg[j];
9351                                 uint8_t *m = t->structs[p->metadata_struct_id];
9352
9353                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9354                         }
9355                 }
9356         }
9357
9358         return 0;
9359
9360 error:
9361         learner_build_free(p);
9362         return status;
9363 }
9364
9365 static void
9366 learner_free(struct rte_swx_pipeline *p)
9367 {
9368         learner_build_free(p);
9369
9370         /* Learner tables. */
9371         for ( ; ; ) {
9372                 struct learner *l;
9373
9374                 l = TAILQ_FIRST(&p->learners);
9375                 if (!l)
9376                         break;
9377
9378                 TAILQ_REMOVE(&p->learners, l, node);
9379                 free(l->fields);
9380                 free(l->actions);
9381                 free(l->action_arg);
9382                 free(l->default_action_data);
9383                 free(l);
9384         }
9385 }
9386
9387 /*
9388  * Table state.
9389  */
9390 static int
9391 table_state_build(struct rte_swx_pipeline *p)
9392 {
9393         struct table *table;
9394         struct selector *s;
9395         struct learner *l;
9396
9397         p->table_state = calloc(p->n_tables + p->n_selectors,
9398                                 sizeof(struct rte_swx_table_state));
9399         CHECK(p->table_state, ENOMEM);
9400
9401         TAILQ_FOREACH(table, &p->tables, node) {
9402                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9403
9404                 if (table->type) {
9405                         struct rte_swx_table_params *params;
9406
9407                         /* ts->obj. */
9408                         params = table_params_get(table);
9409                         CHECK(params, ENOMEM);
9410
9411                         ts->obj = table->type->ops.create(params,
9412                                 NULL,
9413                                 table->args,
9414                                 p->numa_node);
9415
9416                         table_params_free(params);
9417                         CHECK(ts->obj, ENODEV);
9418                 }
9419
9420                 /* ts->default_action_data. */
9421                 if (table->action_data_size_max) {
9422                         ts->default_action_data =
9423                                 malloc(table->action_data_size_max);
9424                         CHECK(ts->default_action_data, ENOMEM);
9425
9426                         memcpy(ts->default_action_data,
9427                                table->default_action_data,
9428                                table->action_data_size_max);
9429                 }
9430
9431                 /* ts->default_action_id. */
9432                 ts->default_action_id = table->default_action->id;
9433         }
9434
9435         TAILQ_FOREACH(s, &p->selectors, node) {
9436                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9437                 struct rte_swx_table_selector_params *params;
9438
9439                 /* ts->obj. */
9440                 params = selector_table_params_get(s);
9441                 CHECK(params, ENOMEM);
9442
9443                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9444
9445                 selector_params_free(params);
9446                 CHECK(ts->obj, ENODEV);
9447         }
9448
9449         TAILQ_FOREACH(l, &p->learners, node) {
9450                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9451                         p->n_selectors + l->id];
9452                 struct rte_swx_table_learner_params *params;
9453
9454                 /* ts->obj. */
9455                 params = learner_params_get(l);
9456                 CHECK(params, ENOMEM);
9457
9458                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9459                 learner_params_free(params);
9460                 CHECK(ts->obj, ENODEV);
9461
9462                 /* ts->default_action_data. */
9463                 if (l->action_data_size_max) {
9464                         ts->default_action_data = malloc(l->action_data_size_max);
9465                         CHECK(ts->default_action_data, ENOMEM);
9466
9467                         memcpy(ts->default_action_data,
9468                                l->default_action_data,
9469                                l->action_data_size_max);
9470                 }
9471
9472                 /* ts->default_action_id. */
9473                 ts->default_action_id = l->default_action->id;
9474         }
9475
9476         return 0;
9477 }
9478
9479 static void
9480 table_state_build_free(struct rte_swx_pipeline *p)
9481 {
9482         uint32_t i;
9483
9484         if (!p->table_state)
9485                 return;
9486
9487         for (i = 0; i < p->n_tables; i++) {
9488                 struct rte_swx_table_state *ts = &p->table_state[i];
9489                 struct table *table = table_find_by_id(p, i);
9490
9491                 /* ts->obj. */
9492                 if (table->type && ts->obj)
9493                         table->type->ops.free(ts->obj);
9494
9495                 /* ts->default_action_data. */
9496                 free(ts->default_action_data);
9497         }
9498
9499         for (i = 0; i < p->n_selectors; i++) {
9500                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9501
9502                 /* ts->obj. */
9503                 if (ts->obj)
9504                         rte_swx_table_selector_free(ts->obj);
9505         }
9506
9507         for (i = 0; i < p->n_learners; i++) {
9508                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9509
9510                 /* ts->obj. */
9511                 if (ts->obj)
9512                         rte_swx_table_learner_free(ts->obj);
9513
9514                 /* ts->default_action_data. */
9515                 free(ts->default_action_data);
9516         }
9517
9518         free(p->table_state);
9519         p->table_state = NULL;
9520 }
9521
9522 static void
9523 table_state_free(struct rte_swx_pipeline *p)
9524 {
9525         table_state_build_free(p);
9526 }
9527
9528 /*
9529  * Register array.
9530  */
9531 static struct regarray *
9532 regarray_find(struct rte_swx_pipeline *p, const char *name)
9533 {
9534         struct regarray *elem;
9535
9536         TAILQ_FOREACH(elem, &p->regarrays, node)
9537                 if (!strcmp(elem->name, name))
9538                         return elem;
9539
9540         return NULL;
9541 }
9542
9543 static struct regarray *
9544 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9545 {
9546         struct regarray *elem = NULL;
9547
9548         TAILQ_FOREACH(elem, &p->regarrays, node)
9549                 if (elem->id == id)
9550                         return elem;
9551
9552         return NULL;
9553 }
9554
9555 int
9556 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9557                               const char *name,
9558                               uint32_t size,
9559                               uint64_t init_val)
9560 {
9561         struct regarray *r;
9562
9563         CHECK(p, EINVAL);
9564
9565         CHECK_NAME(name, EINVAL);
9566         CHECK(!regarray_find(p, name), EEXIST);
9567
9568         CHECK(size, EINVAL);
9569         size = rte_align32pow2(size);
9570
9571         /* Memory allocation. */
9572         r = calloc(1, sizeof(struct regarray));
9573         CHECK(r, ENOMEM);
9574
9575         /* Node initialization. */
9576         strcpy(r->name, name);
9577         r->init_val = init_val;
9578         r->size = size;
9579         r->id = p->n_regarrays;
9580
9581         /* Node add to tailq. */
9582         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9583         p->n_regarrays++;
9584
9585         return 0;
9586 }
9587
9588 static int
9589 regarray_build(struct rte_swx_pipeline *p)
9590 {
9591         struct regarray *regarray;
9592
9593         if (!p->n_regarrays)
9594                 return 0;
9595
9596         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9597         CHECK(p->regarray_runtime, ENOMEM);
9598
9599         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9600                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9601                 uint32_t i;
9602
9603                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9604                                          RTE_CACHE_LINE_SIZE,
9605                                          p->numa_node);
9606                 CHECK(r->regarray, ENOMEM);
9607
9608                 if (regarray->init_val)
9609                         for (i = 0; i < regarray->size; i++)
9610                                 r->regarray[i] = regarray->init_val;
9611
9612                 r->size_mask = regarray->size - 1;
9613         }
9614
9615         return 0;
9616 }
9617
9618 static void
9619 regarray_build_free(struct rte_swx_pipeline *p)
9620 {
9621         uint32_t i;
9622
9623         if (!p->regarray_runtime)
9624                 return;
9625
9626         for (i = 0; i < p->n_regarrays; i++) {
9627                 struct regarray *regarray = regarray_find_by_id(p, i);
9628                 struct regarray_runtime *r = &p->regarray_runtime[i];
9629
9630                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9631         }
9632
9633         free(p->regarray_runtime);
9634         p->regarray_runtime = NULL;
9635 }
9636
9637 static void
9638 regarray_free(struct rte_swx_pipeline *p)
9639 {
9640         regarray_build_free(p);
9641
9642         for ( ; ; ) {
9643                 struct regarray *elem;
9644
9645                 elem = TAILQ_FIRST(&p->regarrays);
9646                 if (!elem)
9647                         break;
9648
9649                 TAILQ_REMOVE(&p->regarrays, elem, node);
9650                 free(elem);
9651         }
9652 }
9653
9654 /*
9655  * Meter array.
9656  */
9657 static struct meter_profile *
9658 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9659 {
9660         struct meter_profile *elem;
9661
9662         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9663                 if (!strcmp(elem->name, name))
9664                         return elem;
9665
9666         return NULL;
9667 }
9668
9669 static struct metarray *
9670 metarray_find(struct rte_swx_pipeline *p, const char *name)
9671 {
9672         struct metarray *elem;
9673
9674         TAILQ_FOREACH(elem, &p->metarrays, node)
9675                 if (!strcmp(elem->name, name))
9676                         return elem;
9677
9678         return NULL;
9679 }
9680
9681 static struct metarray *
9682 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9683 {
9684         struct metarray *elem = NULL;
9685
9686         TAILQ_FOREACH(elem, &p->metarrays, node)
9687                 if (elem->id == id)
9688                         return elem;
9689
9690         return NULL;
9691 }
9692
9693 int
9694 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9695                                  const char *name,
9696                                  uint32_t size)
9697 {
9698         struct metarray *m;
9699
9700         CHECK(p, EINVAL);
9701
9702         CHECK_NAME(name, EINVAL);
9703         CHECK(!metarray_find(p, name), EEXIST);
9704
9705         CHECK(size, EINVAL);
9706         size = rte_align32pow2(size);
9707
9708         /* Memory allocation. */
9709         m = calloc(1, sizeof(struct metarray));
9710         CHECK(m, ENOMEM);
9711
9712         /* Node initialization. */
9713         strcpy(m->name, name);
9714         m->size = size;
9715         m->id = p->n_metarrays;
9716
9717         /* Node add to tailq. */
9718         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9719         p->n_metarrays++;
9720
9721         return 0;
9722 }
9723
9724 struct meter_profile meter_profile_default = {
9725         .node = {0},
9726         .name = "",
9727         .params = {0},
9728
9729         .profile = {
9730                 .cbs = 10000,
9731                 .pbs = 10000,
9732                 .cir_period = 1,
9733                 .cir_bytes_per_period = 1,
9734                 .pir_period = 1,
9735                 .pir_bytes_per_period = 1,
9736         },
9737
9738         .n_users = 0,
9739 };
9740
9741 static void
9742 meter_init(struct meter *m)
9743 {
9744         memset(m, 0, sizeof(struct meter));
9745         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9746         m->profile = &meter_profile_default;
9747         m->color_mask = RTE_COLOR_GREEN;
9748
9749         meter_profile_default.n_users++;
9750 }
9751
9752 static int
9753 metarray_build(struct rte_swx_pipeline *p)
9754 {
9755         struct metarray *m;
9756
9757         if (!p->n_metarrays)
9758                 return 0;
9759
9760         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9761         CHECK(p->metarray_runtime, ENOMEM);
9762
9763         TAILQ_FOREACH(m, &p->metarrays, node) {
9764                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9765                 uint32_t i;
9766
9767                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9768                                          RTE_CACHE_LINE_SIZE,
9769                                          p->numa_node);
9770                 CHECK(r->metarray, ENOMEM);
9771
9772                 for (i = 0; i < m->size; i++)
9773                         meter_init(&r->metarray[i]);
9774
9775                 r->size_mask = m->size - 1;
9776         }
9777
9778         return 0;
9779 }
9780
9781 static void
9782 metarray_build_free(struct rte_swx_pipeline *p)
9783 {
9784         uint32_t i;
9785
9786         if (!p->metarray_runtime)
9787                 return;
9788
9789         for (i = 0; i < p->n_metarrays; i++) {
9790                 struct metarray *m = metarray_find_by_id(p, i);
9791                 struct metarray_runtime *r = &p->metarray_runtime[i];
9792
9793                 env_free(r->metarray, m->size * sizeof(struct meter));
9794         }
9795
9796         free(p->metarray_runtime);
9797         p->metarray_runtime = NULL;
9798 }
9799
9800 static void
9801 metarray_free(struct rte_swx_pipeline *p)
9802 {
9803         metarray_build_free(p);
9804
9805         /* Meter arrays. */
9806         for ( ; ; ) {
9807                 struct metarray *elem;
9808
9809                 elem = TAILQ_FIRST(&p->metarrays);
9810                 if (!elem)
9811                         break;
9812
9813                 TAILQ_REMOVE(&p->metarrays, elem, node);
9814                 free(elem);
9815         }
9816
9817         /* Meter profiles. */
9818         for ( ; ; ) {
9819                 struct meter_profile *elem;
9820
9821                 elem = TAILQ_FIRST(&p->meter_profiles);
9822                 if (!elem)
9823                         break;
9824
9825                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9826                 free(elem);
9827         }
9828 }
9829
9830 /*
9831  * Pipeline.
9832  */
9833 int
9834 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9835 {
9836         struct rte_swx_pipeline *pipeline;
9837
9838         /* Check input parameters. */
9839         CHECK(p, EINVAL);
9840
9841         /* Memory allocation. */
9842         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9843         CHECK(pipeline, ENOMEM);
9844
9845         /* Initialization. */
9846         TAILQ_INIT(&pipeline->struct_types);
9847         TAILQ_INIT(&pipeline->port_in_types);
9848         TAILQ_INIT(&pipeline->ports_in);
9849         TAILQ_INIT(&pipeline->port_out_types);
9850         TAILQ_INIT(&pipeline->ports_out);
9851         TAILQ_INIT(&pipeline->extern_types);
9852         TAILQ_INIT(&pipeline->extern_objs);
9853         TAILQ_INIT(&pipeline->extern_funcs);
9854         TAILQ_INIT(&pipeline->headers);
9855         TAILQ_INIT(&pipeline->actions);
9856         TAILQ_INIT(&pipeline->table_types);
9857         TAILQ_INIT(&pipeline->tables);
9858         TAILQ_INIT(&pipeline->selectors);
9859         TAILQ_INIT(&pipeline->learners);
9860         TAILQ_INIT(&pipeline->regarrays);
9861         TAILQ_INIT(&pipeline->meter_profiles);
9862         TAILQ_INIT(&pipeline->metarrays);
9863
9864         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9865         pipeline->numa_node = numa_node;
9866
9867         *p = pipeline;
9868         return 0;
9869 }
9870
9871 void
9872 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9873 {
9874         if (!p)
9875                 return;
9876
9877         free(p->instructions);
9878
9879         metarray_free(p);
9880         regarray_free(p);
9881         table_state_free(p);
9882         learner_free(p);
9883         selector_free(p);
9884         table_free(p);
9885         action_free(p);
9886         metadata_free(p);
9887         header_free(p);
9888         extern_func_free(p);
9889         extern_obj_free(p);
9890         port_out_free(p);
9891         port_in_free(p);
9892         struct_free(p);
9893
9894         free(p);
9895 }
9896
9897 int
9898 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9899                                      const char **instructions,
9900                                      uint32_t n_instructions)
9901 {
9902         int err;
9903         uint32_t i;
9904
9905         err = instruction_config(p, NULL, instructions, n_instructions);
9906         if (err)
9907                 return err;
9908
9909         /* Thread instruction pointer reset. */
9910         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9911                 struct thread *t = &p->threads[i];
9912
9913                 thread_ip_reset(p, t);
9914         }
9915
9916         return 0;
9917 }
9918
9919 int
9920 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9921 {
9922         int status;
9923
9924         CHECK(p, EINVAL);
9925         CHECK(p->build_done == 0, EEXIST);
9926
9927         status = port_in_build(p);
9928         if (status)
9929                 goto error;
9930
9931         status = port_out_build(p);
9932         if (status)
9933                 goto error;
9934
9935         status = struct_build(p);
9936         if (status)
9937                 goto error;
9938
9939         status = extern_obj_build(p);
9940         if (status)
9941                 goto error;
9942
9943         status = extern_func_build(p);
9944         if (status)
9945                 goto error;
9946
9947         status = header_build(p);
9948         if (status)
9949                 goto error;
9950
9951         status = metadata_build(p);
9952         if (status)
9953                 goto error;
9954
9955         status = action_build(p);
9956         if (status)
9957                 goto error;
9958
9959         status = table_build(p);
9960         if (status)
9961                 goto error;
9962
9963         status = selector_build(p);
9964         if (status)
9965                 goto error;
9966
9967         status = learner_build(p);
9968         if (status)
9969                 goto error;
9970
9971         status = table_state_build(p);
9972         if (status)
9973                 goto error;
9974
9975         status = regarray_build(p);
9976         if (status)
9977                 goto error;
9978
9979         status = metarray_build(p);
9980         if (status)
9981                 goto error;
9982
9983         p->build_done = 1;
9984         return 0;
9985
9986 error:
9987         metarray_build_free(p);
9988         regarray_build_free(p);
9989         table_state_build_free(p);
9990         learner_build_free(p);
9991         selector_build_free(p);
9992         table_build_free(p);
9993         action_build_free(p);
9994         metadata_build_free(p);
9995         header_build_free(p);
9996         extern_func_build_free(p);
9997         extern_obj_build_free(p);
9998         port_out_build_free(p);
9999         port_in_build_free(p);
10000         struct_build_free(p);
10001
10002         return status;
10003 }
10004
10005 void
10006 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10007 {
10008         uint32_t i;
10009
10010         for (i = 0; i < n_instructions; i++)
10011                 instr_exec(p);
10012 }
10013
10014 void
10015 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10016 {
10017         uint32_t i;
10018
10019         for (i = 0; i < p->n_ports_out; i++) {
10020                 struct port_out_runtime *port = &p->out[i];
10021
10022                 if (port->flush)
10023                         port->flush(port->obj);
10024         }
10025 }
10026
10027 /*
10028  * Control.
10029  */
10030 int
10031 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10032                               struct rte_swx_ctl_pipeline_info *pipeline)
10033 {
10034         struct action *action;
10035         struct table *table;
10036         uint32_t n_actions = 0, n_tables = 0;
10037
10038         if (!p || !pipeline)
10039                 return -EINVAL;
10040
10041         TAILQ_FOREACH(action, &p->actions, node)
10042                 n_actions++;
10043
10044         TAILQ_FOREACH(table, &p->tables, node)
10045                 n_tables++;
10046
10047         pipeline->n_ports_in = p->n_ports_in;
10048         pipeline->n_ports_out = p->n_ports_out;
10049         pipeline->n_actions = n_actions;
10050         pipeline->n_tables = n_tables;
10051         pipeline->n_selectors = p->n_selectors;
10052         pipeline->n_learners = p->n_learners;
10053         pipeline->n_regarrays = p->n_regarrays;
10054         pipeline->n_metarrays = p->n_metarrays;
10055
10056         return 0;
10057 }
10058
10059 int
10060 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10061 {
10062         if (!p || !numa_node)
10063                 return -EINVAL;
10064
10065         *numa_node = p->numa_node;
10066         return 0;
10067 }
10068
10069 int
10070 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10071                             uint32_t action_id,
10072                             struct rte_swx_ctl_action_info *action)
10073 {
10074         struct action *a = NULL;
10075
10076         if (!p || (action_id >= p->n_actions) || !action)
10077                 return -EINVAL;
10078
10079         a = action_find_by_id(p, action_id);
10080         if (!a)
10081                 return -EINVAL;
10082
10083         strcpy(action->name, a->name);
10084         action->n_args = a->st ? a->st->n_fields : 0;
10085         return 0;
10086 }
10087
10088 int
10089 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10090                                 uint32_t action_id,
10091                                 uint32_t action_arg_id,
10092                                 struct rte_swx_ctl_action_arg_info *action_arg)
10093 {
10094         struct action *a = NULL;
10095         struct field *arg = NULL;
10096
10097         if (!p || (action_id >= p->n_actions) || !action_arg)
10098                 return -EINVAL;
10099
10100         a = action_find_by_id(p, action_id);
10101         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10102                 return -EINVAL;
10103
10104         arg = &a->st->fields[action_arg_id];
10105         strcpy(action_arg->name, arg->name);
10106         action_arg->n_bits = arg->n_bits;
10107         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10108
10109         return 0;
10110 }
10111
10112 int
10113 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10114                            uint32_t table_id,
10115                            struct rte_swx_ctl_table_info *table)
10116 {
10117         struct table *t = NULL;
10118
10119         if (!p || !table)
10120                 return -EINVAL;
10121
10122         t = table_find_by_id(p, table_id);
10123         if (!t)
10124                 return -EINVAL;
10125
10126         strcpy(table->name, t->name);
10127         strcpy(table->args, t->args);
10128         table->n_match_fields = t->n_fields;
10129         table->n_actions = t->n_actions;
10130         table->default_action_is_const = t->default_action_is_const;
10131         table->size = t->size;
10132         return 0;
10133 }
10134
10135 int
10136 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10137         uint32_t table_id,
10138         uint32_t match_field_id,
10139         struct rte_swx_ctl_table_match_field_info *match_field)
10140 {
10141         struct table *t;
10142         struct match_field *f;
10143
10144         if (!p || (table_id >= p->n_tables) || !match_field)
10145                 return -EINVAL;
10146
10147         t = table_find_by_id(p, table_id);
10148         if (!t || (match_field_id >= t->n_fields))
10149                 return -EINVAL;
10150
10151         f = &t->fields[match_field_id];
10152         match_field->match_type = f->match_type;
10153         match_field->is_header = t->header ? 1 : 0;
10154         match_field->n_bits = f->field->n_bits;
10155         match_field->offset = f->field->offset;
10156
10157         return 0;
10158 }
10159
10160 int
10161 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10162         uint32_t table_id,
10163         uint32_t table_action_id,
10164         struct rte_swx_ctl_table_action_info *table_action)
10165 {
10166         struct table *t;
10167
10168         if (!p || (table_id >= p->n_tables) || !table_action)
10169                 return -EINVAL;
10170
10171         t = table_find_by_id(p, table_id);
10172         if (!t || (table_action_id >= t->n_actions))
10173                 return -EINVAL;
10174
10175         table_action->action_id = t->actions[table_action_id]->id;
10176
10177         return 0;
10178 }
10179
10180 int
10181 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10182                           uint32_t table_id,
10183                           struct rte_swx_table_ops *table_ops,
10184                           int *is_stub)
10185 {
10186         struct table *t;
10187
10188         if (!p || (table_id >= p->n_tables))
10189                 return -EINVAL;
10190
10191         t = table_find_by_id(p, table_id);
10192         if (!t)
10193                 return -EINVAL;
10194
10195         if (t->type) {
10196                 if (table_ops)
10197                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10198                 *is_stub = 0;
10199         } else {
10200                 *is_stub = 1;
10201         }
10202
10203         return 0;
10204 }
10205
10206 int
10207 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10208                               uint32_t selector_id,
10209                               struct rte_swx_ctl_selector_info *selector)
10210 {
10211         struct selector *s = NULL;
10212
10213         if (!p || !selector)
10214                 return -EINVAL;
10215
10216         s = selector_find_by_id(p, selector_id);
10217         if (!s)
10218                 return -EINVAL;
10219
10220         strcpy(selector->name, s->name);
10221
10222         selector->n_selector_fields = s->n_selector_fields;
10223         selector->n_groups_max = s->n_groups_max;
10224         selector->n_members_per_group_max = s->n_members_per_group_max;
10225
10226         return 0;
10227 }
10228
10229 int
10230 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10231          uint32_t selector_id,
10232          struct rte_swx_ctl_table_match_field_info *field)
10233 {
10234         struct selector *s;
10235
10236         if (!p || (selector_id >= p->n_selectors) || !field)
10237                 return -EINVAL;
10238
10239         s = selector_find_by_id(p, selector_id);
10240         if (!s)
10241                 return -EINVAL;
10242
10243         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10244         field->is_header = 0;
10245         field->n_bits = s->group_id_field->n_bits;
10246         field->offset = s->group_id_field->offset;
10247
10248         return 0;
10249 }
10250
10251 int
10252 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10253          uint32_t selector_id,
10254          uint32_t selector_field_id,
10255          struct rte_swx_ctl_table_match_field_info *field)
10256 {
10257         struct selector *s;
10258         struct field *f;
10259
10260         if (!p || (selector_id >= p->n_selectors) || !field)
10261                 return -EINVAL;
10262
10263         s = selector_find_by_id(p, selector_id);
10264         if (!s || (selector_field_id >= s->n_selector_fields))
10265                 return -EINVAL;
10266
10267         f = s->selector_fields[selector_field_id];
10268         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10269         field->is_header = s->selector_header ? 1 : 0;
10270         field->n_bits = f->n_bits;
10271         field->offset = f->offset;
10272
10273         return 0;
10274 }
10275
10276 int
10277 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10278          uint32_t selector_id,
10279          struct rte_swx_ctl_table_match_field_info *field)
10280 {
10281         struct selector *s;
10282
10283         if (!p || (selector_id >= p->n_selectors) || !field)
10284                 return -EINVAL;
10285
10286         s = selector_find_by_id(p, selector_id);
10287         if (!s)
10288                 return -EINVAL;
10289
10290         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10291         field->is_header = 0;
10292         field->n_bits = s->member_id_field->n_bits;
10293         field->offset = s->member_id_field->offset;
10294
10295         return 0;
10296 }
10297
10298 int
10299 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10300                              uint32_t learner_id,
10301                              struct rte_swx_ctl_learner_info *learner)
10302 {
10303         struct learner *l = NULL;
10304
10305         if (!p || !learner)
10306                 return -EINVAL;
10307
10308         l = learner_find_by_id(p, learner_id);
10309         if (!l)
10310                 return -EINVAL;
10311
10312         strcpy(learner->name, l->name);
10313
10314         learner->n_match_fields = l->n_fields;
10315         learner->n_actions = l->n_actions;
10316         learner->default_action_is_const = l->default_action_is_const;
10317         learner->size = l->size;
10318
10319         return 0;
10320 }
10321
10322 int
10323 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10324                                          uint32_t learner_id,
10325                                          uint32_t match_field_id,
10326                                          struct rte_swx_ctl_table_match_field_info *match_field)
10327 {
10328         struct learner *l;
10329         struct field *f;
10330
10331         if (!p || (learner_id >= p->n_learners) || !match_field)
10332                 return -EINVAL;
10333
10334         l = learner_find_by_id(p, learner_id);
10335         if (!l || (match_field_id >= l->n_fields))
10336                 return -EINVAL;
10337
10338         f = l->fields[match_field_id];
10339         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10340         match_field->is_header = l->header ? 1 : 0;
10341         match_field->n_bits = f->n_bits;
10342         match_field->offset = f->offset;
10343
10344         return 0;
10345 }
10346
10347 int
10348 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10349                                     uint32_t learner_id,
10350                                     uint32_t learner_action_id,
10351                                     struct rte_swx_ctl_table_action_info *learner_action)
10352 {
10353         struct learner *l;
10354
10355         if (!p || (learner_id >= p->n_learners) || !learner_action)
10356                 return -EINVAL;
10357
10358         l = learner_find_by_id(p, learner_id);
10359         if (!l || (learner_action_id >= l->n_actions))
10360                 return -EINVAL;
10361
10362         learner_action->action_id = l->actions[learner_action_id]->id;
10363
10364         return 0;
10365 }
10366
10367 int
10368 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10369                                  struct rte_swx_table_state **table_state)
10370 {
10371         if (!p || !table_state || !p->build_done)
10372                 return -EINVAL;
10373
10374         *table_state = p->table_state;
10375         return 0;
10376 }
10377
10378 int
10379 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10380                                  struct rte_swx_table_state *table_state)
10381 {
10382         if (!p || !table_state || !p->build_done)
10383                 return -EINVAL;
10384
10385         p->table_state = table_state;
10386         return 0;
10387 }
10388
10389 int
10390 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10391                                         uint32_t port_id,
10392                                         struct rte_swx_port_in_stats *stats)
10393 {
10394         struct port_in *port;
10395
10396         if (!p || !stats)
10397                 return -EINVAL;
10398
10399         port = port_in_find(p, port_id);
10400         if (!port)
10401                 return -EINVAL;
10402
10403         port->type->ops.stats_read(port->obj, stats);
10404         return 0;
10405 }
10406
10407 int
10408 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10409                                          uint32_t port_id,
10410                                          struct rte_swx_port_out_stats *stats)
10411 {
10412         struct port_out *port;
10413
10414         if (!p || !stats)
10415                 return -EINVAL;
10416
10417         port = port_out_find(p, port_id);
10418         if (!port)
10419                 return -EINVAL;
10420
10421         port->type->ops.stats_read(port->obj, stats);
10422         return 0;
10423 }
10424
10425 int
10426 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10427                                       const char *table_name,
10428                                       struct rte_swx_table_stats *stats)
10429 {
10430         struct table *table;
10431         struct table_statistics *table_stats;
10432
10433         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10434                 return -EINVAL;
10435
10436         table = table_find(p, table_name);
10437         if (!table)
10438                 return -EINVAL;
10439
10440         table_stats = &p->table_stats[table->id];
10441
10442         memcpy(stats->n_pkts_action,
10443                table_stats->n_pkts_action,
10444                p->n_actions * sizeof(uint64_t));
10445
10446         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10447         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10448
10449         return 0;
10450 }
10451
10452 int
10453 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10454         const char *selector_name,
10455         struct rte_swx_pipeline_selector_stats *stats)
10456 {
10457         struct selector *s;
10458
10459         if (!p || !selector_name || !selector_name[0] || !stats)
10460                 return -EINVAL;
10461
10462         s = selector_find(p, selector_name);
10463         if (!s)
10464                 return -EINVAL;
10465
10466         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10467
10468         return 0;
10469 }
10470
10471 int
10472 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10473                                         const char *learner_name,
10474                                         struct rte_swx_learner_stats *stats)
10475 {
10476         struct learner *l;
10477         struct learner_statistics *learner_stats;
10478
10479         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10480                 return -EINVAL;
10481
10482         l = learner_find(p, learner_name);
10483         if (!l)
10484                 return -EINVAL;
10485
10486         learner_stats = &p->learner_stats[l->id];
10487
10488         memcpy(stats->n_pkts_action,
10489                learner_stats->n_pkts_action,
10490                p->n_actions * sizeof(uint64_t));
10491
10492         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10493         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10494
10495         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10496         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10497
10498         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10499
10500         return 0;
10501 }
10502
10503 int
10504 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10505                               uint32_t regarray_id,
10506                               struct rte_swx_ctl_regarray_info *regarray)
10507 {
10508         struct regarray *r;
10509
10510         if (!p || !regarray)
10511                 return -EINVAL;
10512
10513         r = regarray_find_by_id(p, regarray_id);
10514         if (!r)
10515                 return -EINVAL;
10516
10517         strcpy(regarray->name, r->name);
10518         regarray->size = r->size;
10519         return 0;
10520 }
10521
10522 int
10523 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10524                                    const char *regarray_name,
10525                                    uint32_t regarray_index,
10526                                    uint64_t *value)
10527 {
10528         struct regarray *regarray;
10529         struct regarray_runtime *r;
10530
10531         if (!p || !regarray_name || !value)
10532                 return -EINVAL;
10533
10534         regarray = regarray_find(p, regarray_name);
10535         if (!regarray || (regarray_index >= regarray->size))
10536                 return -EINVAL;
10537
10538         r = &p->regarray_runtime[regarray->id];
10539         *value = r->regarray[regarray_index];
10540         return 0;
10541 }
10542
10543 int
10544 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10545                                    const char *regarray_name,
10546                                    uint32_t regarray_index,
10547                                    uint64_t value)
10548 {
10549         struct regarray *regarray;
10550         struct regarray_runtime *r;
10551
10552         if (!p || !regarray_name)
10553                 return -EINVAL;
10554
10555         regarray = regarray_find(p, regarray_name);
10556         if (!regarray || (regarray_index >= regarray->size))
10557                 return -EINVAL;
10558
10559         r = &p->regarray_runtime[regarray->id];
10560         r->regarray[regarray_index] = value;
10561         return 0;
10562 }
10563
10564 int
10565 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10566                               uint32_t metarray_id,
10567                               struct rte_swx_ctl_metarray_info *metarray)
10568 {
10569         struct metarray *m;
10570
10571         if (!p || !metarray)
10572                 return -EINVAL;
10573
10574         m = metarray_find_by_id(p, metarray_id);
10575         if (!m)
10576                 return -EINVAL;
10577
10578         strcpy(metarray->name, m->name);
10579         metarray->size = m->size;
10580         return 0;
10581 }
10582
10583 int
10584 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10585                               const char *name,
10586                               struct rte_meter_trtcm_params *params)
10587 {
10588         struct meter_profile *mp;
10589         int status;
10590
10591         CHECK(p, EINVAL);
10592         CHECK_NAME(name, EINVAL);
10593         CHECK(params, EINVAL);
10594         CHECK(!meter_profile_find(p, name), EEXIST);
10595
10596         /* Node allocation. */
10597         mp = calloc(1, sizeof(struct meter_profile));
10598         CHECK(mp, ENOMEM);
10599
10600         /* Node initialization. */
10601         strcpy(mp->name, name);
10602         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10603         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10604         if (status) {
10605                 free(mp);
10606                 CHECK(0, EINVAL);
10607         }
10608
10609         /* Node add to tailq. */
10610         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10611
10612         return 0;
10613 }
10614
10615 int
10616 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10617                                  const char *name)
10618 {
10619         struct meter_profile *mp;
10620
10621         CHECK(p, EINVAL);
10622         CHECK_NAME(name, EINVAL);
10623
10624         mp = meter_profile_find(p, name);
10625         CHECK(mp, EINVAL);
10626         CHECK(!mp->n_users, EBUSY);
10627
10628         /* Remove node from tailq. */
10629         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10630         free(mp);
10631
10632         return 0;
10633 }
10634
10635 int
10636 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10637                         const char *metarray_name,
10638                         uint32_t metarray_index)
10639 {
10640         struct meter_profile *mp_old;
10641         struct metarray *metarray;
10642         struct metarray_runtime *metarray_runtime;
10643         struct meter *m;
10644
10645         CHECK(p, EINVAL);
10646         CHECK_NAME(metarray_name, EINVAL);
10647
10648         metarray = metarray_find(p, metarray_name);
10649         CHECK(metarray, EINVAL);
10650         CHECK(metarray_index < metarray->size, EINVAL);
10651
10652         metarray_runtime = &p->metarray_runtime[metarray->id];
10653         m = &metarray_runtime->metarray[metarray_index];
10654         mp_old = m->profile;
10655
10656         meter_init(m);
10657
10658         mp_old->n_users--;
10659
10660         return 0;
10661 }
10662
10663 int
10664 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10665                       const char *metarray_name,
10666                       uint32_t metarray_index,
10667                       const char *profile_name)
10668 {
10669         struct meter_profile *mp, *mp_old;
10670         struct metarray *metarray;
10671         struct metarray_runtime *metarray_runtime;
10672         struct meter *m;
10673
10674         CHECK(p, EINVAL);
10675         CHECK_NAME(metarray_name, EINVAL);
10676
10677         metarray = metarray_find(p, metarray_name);
10678         CHECK(metarray, EINVAL);
10679         CHECK(metarray_index < metarray->size, EINVAL);
10680
10681         mp = meter_profile_find(p, profile_name);
10682         CHECK(mp, EINVAL);
10683
10684         metarray_runtime = &p->metarray_runtime[metarray->id];
10685         m = &metarray_runtime->metarray[metarray_index];
10686         mp_old = m->profile;
10687
10688         memset(m, 0, sizeof(struct meter));
10689         rte_meter_trtcm_config(&m->m, &mp->profile);
10690         m->profile = mp;
10691         m->color_mask = RTE_COLORS;
10692
10693         mp->n_users++;
10694         mp_old->n_users--;
10695
10696         return 0;
10697 }
10698
10699 int
10700 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10701                              const char *metarray_name,
10702                              uint32_t metarray_index,
10703                              struct rte_swx_ctl_meter_stats *stats)
10704 {
10705         struct metarray *metarray;
10706         struct metarray_runtime *metarray_runtime;
10707         struct meter *m;
10708
10709         CHECK(p, EINVAL);
10710         CHECK_NAME(metarray_name, EINVAL);
10711
10712         metarray = metarray_find(p, metarray_name);
10713         CHECK(metarray, EINVAL);
10714         CHECK(metarray_index < metarray->size, EINVAL);
10715
10716         CHECK(stats, EINVAL);
10717
10718         metarray_runtime = &p->metarray_runtime[metarray->id];
10719         m = &metarray_runtime->metarray[metarray_index];
10720
10721         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10722         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10723
10724         return 0;
10725 }