lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t selector_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107         struct selector_runtime *selector = &t->selectors[selector_id];
2108         struct selector_statistics *stats = &p->selector_stats[selector_id];
2109         uint64_t n_pkts = stats->n_pkts;
2110         int done;
2111
2112         /* Table. */
2113         done = rte_swx_table_selector_select(ts->obj,
2114                            selector->mailbox,
2115                            selector->group_id_buffer,
2116                            selector->selector_buffer,
2117                            selector->member_id_buffer);
2118         if (!done) {
2119                 /* Thread. */
2120                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2121                       p->thread_id,
2122                       selector_id);
2123
2124                 thread_yield(p);
2125                 return;
2126         }
2127
2128
2129         TRACE("[Thread %2u] selector %u\n",
2130               p->thread_id,
2131               selector_id);
2132
2133         stats->n_pkts = n_pkts + 1;
2134
2135         /* Thread. */
2136         thread_ip_inc(p);
2137 }
2138
2139 static inline void
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t learner_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146                 p->n_selectors + learner_id];
2147         struct learner_runtime *l = &t->learners[learner_id];
2148         struct learner_statistics *stats = &p->learner_stats[learner_id];
2149         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150         uint8_t *action_data;
2151         int done, hit;
2152
2153         /* Table. */
2154         time = rte_get_tsc_cycles();
2155
2156         done = rte_swx_table_learner_lookup(ts->obj,
2157                                             l->mailbox,
2158                                             time,
2159                                             l->key,
2160                                             &action_id,
2161                                             &action_data,
2162                                             &hit);
2163         if (!done) {
2164                 /* Thread. */
2165                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2166                       p->thread_id,
2167                       learner_id);
2168
2169                 thread_yield(p);
2170                 return;
2171         }
2172
2173         action_id = hit ? action_id : ts->default_action_id;
2174         action_data = hit ? action_data : ts->default_action_data;
2175         n_pkts_hit = stats->n_pkts_hit[hit];
2176         n_pkts_action = stats->n_pkts_action[action_id];
2177
2178         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2179               p->thread_id,
2180               learner_id,
2181               hit ? "hit" : "miss",
2182               (uint32_t)action_id);
2183
2184         t->action_id = action_id;
2185         t->structs[0] = action_data;
2186         t->hit = hit;
2187         t->learner_id = learner_id;
2188         t->time = time;
2189         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2191
2192         /* Thread. */
2193         thread_ip_action_call(p, t, action_id);
2194 }
2195
2196 /*
2197  * learn.
2198  */
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2201
2202 static int
2203 action_has_nbo_args(struct action *a);
2204
2205 static int
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207                       struct action *action,
2208                       char **tokens,
2209                       int n_tokens,
2210                       struct instruction *instr,
2211                       struct instruction_data *data __rte_unused)
2212 {
2213         struct action *a;
2214
2215         CHECK(action, EINVAL);
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         a = action_find(p, tokens[1]);
2219         CHECK(a, EINVAL);
2220         CHECK(!action_has_nbo_args(a), EINVAL);
2221
2222         instr->type = INSTR_LEARNER_LEARN;
2223         instr->learn.action_id = a->id;
2224
2225         return 0;
2226 }
2227
2228 static inline void
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233         uint64_t action_id = ip->learn.action_id;
2234         uint32_t learner_id = t->learner_id;
2235         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2236                 p->n_selectors + learner_id];
2237         struct learner_runtime *l = &t->learners[learner_id];
2238         struct learner_statistics *stats = &p->learner_stats[learner_id];
2239         uint32_t status;
2240
2241         /* Table. */
2242         status = rte_swx_table_learner_add(ts->obj,
2243                                            l->mailbox,
2244                                            t->time,
2245                                            action_id,
2246                                            l->action_data[action_id]);
2247
2248         TRACE("[Thread %2u] learner %u learn %s\n",
2249               p->thread_id,
2250               learner_id,
2251               status ? "ok" : "error");
2252
2253         stats->n_pkts_learn[status] += 1;
2254
2255         /* Thread. */
2256         thread_ip_inc(p);
2257 }
2258
2259 /*
2260  * forget.
2261  */
2262 static int
2263 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2264                        struct action *action,
2265                        char **tokens __rte_unused,
2266                        int n_tokens,
2267                        struct instruction *instr,
2268                        struct instruction_data *data __rte_unused)
2269 {
2270         CHECK(action, EINVAL);
2271         CHECK(n_tokens == 1, EINVAL);
2272
2273         instr->type = INSTR_LEARNER_FORGET;
2274
2275         return 0;
2276 }
2277
2278 static inline void
2279 instr_forget_exec(struct rte_swx_pipeline *p)
2280 {
2281         struct thread *t = &p->threads[p->thread_id];
2282         uint32_t learner_id = t->learner_id;
2283         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2284                 p->n_selectors + learner_id];
2285         struct learner_runtime *l = &t->learners[learner_id];
2286         struct learner_statistics *stats = &p->learner_stats[learner_id];
2287
2288         /* Table. */
2289         rte_swx_table_learner_delete(ts->obj, l->mailbox);
2290
2291         TRACE("[Thread %2u] learner %u forget\n",
2292               p->thread_id,
2293               learner_id);
2294
2295         stats->n_pkts_forget += 1;
2296
2297         /* Thread. */
2298         thread_ip_inc(p);
2299 }
2300
2301 /*
2302  * extern.
2303  */
2304 static int
2305 instr_extern_translate(struct rte_swx_pipeline *p,
2306                        struct action *action __rte_unused,
2307                        char **tokens,
2308                        int n_tokens,
2309                        struct instruction *instr,
2310                        struct instruction_data *data __rte_unused)
2311 {
2312         char *token = tokens[1];
2313
2314         CHECK(n_tokens == 2, EINVAL);
2315
2316         if (token[0] == 'e') {
2317                 struct extern_obj *obj;
2318                 struct extern_type_member_func *func;
2319
2320                 func = extern_obj_member_func_parse(p, token, &obj);
2321                 CHECK(func, EINVAL);
2322
2323                 instr->type = INSTR_EXTERN_OBJ;
2324                 instr->ext_obj.ext_obj_id = obj->id;
2325                 instr->ext_obj.func_id = func->id;
2326
2327                 return 0;
2328         }
2329
2330         if (token[0] == 'f') {
2331                 struct extern_func *func;
2332
2333                 func = extern_func_parse(p, token);
2334                 CHECK(func, EINVAL);
2335
2336                 instr->type = INSTR_EXTERN_FUNC;
2337                 instr->ext_func.ext_func_id = func->id;
2338
2339                 return 0;
2340         }
2341
2342         CHECK(0, EINVAL);
2343 }
2344
2345 static inline void
2346 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2347 {
2348         struct thread *t = &p->threads[p->thread_id];
2349         struct instruction *ip = t->ip;
2350         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2351         uint32_t func_id = ip->ext_obj.func_id;
2352         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2353         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2354
2355         TRACE("[Thread %2u] extern obj %u member func %u\n",
2356               p->thread_id,
2357               obj_id,
2358               func_id);
2359
2360         /* Extern object member function execute. */
2361         uint32_t done = func(obj->obj, obj->mailbox);
2362
2363         /* Thread. */
2364         thread_ip_inc_cond(t, done);
2365         thread_yield_cond(p, done ^ 1);
2366 }
2367
2368 static inline void
2369 instr_extern_func_exec(struct rte_swx_pipeline *p)
2370 {
2371         struct thread *t = &p->threads[p->thread_id];
2372         struct instruction *ip = t->ip;
2373         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2374         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2375         rte_swx_extern_func_t func = ext_func->func;
2376
2377         TRACE("[Thread %2u] extern func %u\n",
2378               p->thread_id,
2379               ext_func_id);
2380
2381         /* Extern function execute. */
2382         uint32_t done = func(ext_func->mailbox);
2383
2384         /* Thread. */
2385         thread_ip_inc_cond(t, done);
2386         thread_yield_cond(p, done ^ 1);
2387 }
2388
2389 /*
2390  * mov.
2391  */
2392 static int
2393 instr_mov_translate(struct rte_swx_pipeline *p,
2394                     struct action *action,
2395                     char **tokens,
2396                     int n_tokens,
2397                     struct instruction *instr,
2398                     struct instruction_data *data __rte_unused)
2399 {
2400         char *dst = tokens[1], *src = tokens[2];
2401         struct field *fdst, *fsrc;
2402         uint64_t src_val;
2403         uint32_t dst_struct_id = 0, src_struct_id = 0;
2404
2405         CHECK(n_tokens == 3, EINVAL);
2406
2407         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2408         CHECK(fdst, EINVAL);
2409         CHECK(!fdst->var_size, EINVAL);
2410
2411         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2412         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2413         if (fsrc) {
2414                 CHECK(!fsrc->var_size, EINVAL);
2415
2416                 instr->type = INSTR_MOV;
2417                 if (dst[0] != 'h' && src[0] == 'h')
2418                         instr->type = INSTR_MOV_MH;
2419                 if (dst[0] == 'h' && src[0] != 'h')
2420                         instr->type = INSTR_MOV_HM;
2421                 if (dst[0] == 'h' && src[0] == 'h')
2422                         instr->type = INSTR_MOV_HH;
2423
2424                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2425                 instr->mov.dst.n_bits = fdst->n_bits;
2426                 instr->mov.dst.offset = fdst->offset / 8;
2427                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2428                 instr->mov.src.n_bits = fsrc->n_bits;
2429                 instr->mov.src.offset = fsrc->offset / 8;
2430                 return 0;
2431         }
2432
2433         /* MOV_I. */
2434         src_val = strtoull(src, &src, 0);
2435         CHECK(!src[0], EINVAL);
2436
2437         if (dst[0] == 'h')
2438                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2439
2440         instr->type = INSTR_MOV_I;
2441         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2442         instr->mov.dst.n_bits = fdst->n_bits;
2443         instr->mov.dst.offset = fdst->offset / 8;
2444         instr->mov.src_val = src_val;
2445         return 0;
2446 }
2447
2448 static inline void
2449 instr_mov_exec(struct rte_swx_pipeline *p)
2450 {
2451         struct thread *t = &p->threads[p->thread_id];
2452         struct instruction *ip = t->ip;
2453
2454         TRACE("[Thread %2u] mov\n",
2455               p->thread_id);
2456
2457         MOV(t, ip);
2458
2459         /* Thread. */
2460         thread_ip_inc(p);
2461 }
2462
2463 static inline void
2464 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2465 {
2466         struct thread *t = &p->threads[p->thread_id];
2467         struct instruction *ip = t->ip;
2468
2469         TRACE("[Thread %2u] mov (mh)\n",
2470               p->thread_id);
2471
2472         MOV_MH(t, ip);
2473
2474         /* Thread. */
2475         thread_ip_inc(p);
2476 }
2477
2478 static inline void
2479 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2480 {
2481         struct thread *t = &p->threads[p->thread_id];
2482         struct instruction *ip = t->ip;
2483
2484         TRACE("[Thread %2u] mov (hm)\n",
2485               p->thread_id);
2486
2487         MOV_HM(t, ip);
2488
2489         /* Thread. */
2490         thread_ip_inc(p);
2491 }
2492
2493 static inline void
2494 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2495 {
2496         struct thread *t = &p->threads[p->thread_id];
2497         struct instruction *ip = t->ip;
2498
2499         TRACE("[Thread %2u] mov (hh)\n",
2500               p->thread_id);
2501
2502         MOV_HH(t, ip);
2503
2504         /* Thread. */
2505         thread_ip_inc(p);
2506 }
2507
2508 static inline void
2509 instr_mov_i_exec(struct rte_swx_pipeline *p)
2510 {
2511         struct thread *t = &p->threads[p->thread_id];
2512         struct instruction *ip = t->ip;
2513
2514         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2515               p->thread_id,
2516               ip->mov.src_val);
2517
2518         MOV_I(t, ip);
2519
2520         /* Thread. */
2521         thread_ip_inc(p);
2522 }
2523
2524 /*
2525  * dma.
2526  */
2527 static inline void
2528 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2529
2530 static inline void
2531 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2532 {
2533         struct thread *t = &p->threads[p->thread_id];
2534         struct instruction *ip = t->ip;
2535         uint8_t *action_data = t->structs[0];
2536         uint64_t valid_headers = t->valid_headers;
2537         uint32_t i;
2538
2539         for (i = 0; i < n_dma; i++) {
2540                 uint32_t header_id = ip->dma.dst.header_id[i];
2541                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2542                 uint32_t offset = ip->dma.src.offset[i];
2543                 uint32_t n_bytes = ip->dma.n_bytes[i];
2544
2545                 struct header_runtime *h = &t->headers[header_id];
2546                 uint8_t *h_ptr0 = h->ptr0;
2547                 uint8_t *h_ptr = t->structs[struct_id];
2548
2549                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2550                         h_ptr : h_ptr0;
2551                 void *src = &action_data[offset];
2552
2553                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2554
2555                 /* Headers. */
2556                 memcpy(dst, src, n_bytes);
2557                 t->structs[struct_id] = dst;
2558                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2559         }
2560
2561         t->valid_headers = valid_headers;
2562 }
2563
2564 static inline void
2565 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2566 {
2567         __instr_dma_ht_exec(p, 1);
2568
2569         /* Thread. */
2570         thread_ip_inc(p);
2571 }
2572
2573 static inline void
2574 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2575 {
2576         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2577               p->thread_id);
2578
2579         __instr_dma_ht_exec(p, 2);
2580
2581         /* Thread. */
2582         thread_ip_inc(p);
2583 }
2584
2585 static inline void
2586 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2587 {
2588         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2589               p->thread_id);
2590
2591         __instr_dma_ht_exec(p, 3);
2592
2593         /* Thread. */
2594         thread_ip_inc(p);
2595 }
2596
2597 static inline void
2598 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2599 {
2600         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2601               p->thread_id);
2602
2603         __instr_dma_ht_exec(p, 4);
2604
2605         /* Thread. */
2606         thread_ip_inc(p);
2607 }
2608
2609 static inline void
2610 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2611 {
2612         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2613               p->thread_id);
2614
2615         __instr_dma_ht_exec(p, 5);
2616
2617         /* Thread. */
2618         thread_ip_inc(p);
2619 }
2620
2621 static inline void
2622 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2623 {
2624         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2625               p->thread_id);
2626
2627         __instr_dma_ht_exec(p, 6);
2628
2629         /* Thread. */
2630         thread_ip_inc(p);
2631 }
2632
2633 static inline void
2634 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2635 {
2636         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2637               p->thread_id);
2638
2639         __instr_dma_ht_exec(p, 7);
2640
2641         /* Thread. */
2642         thread_ip_inc(p);
2643 }
2644
2645 static inline void
2646 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2647 {
2648         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2649               p->thread_id);
2650
2651         __instr_dma_ht_exec(p, 8);
2652
2653         /* Thread. */
2654         thread_ip_inc(p);
2655 }
2656
2657 /*
2658  * alu.
2659  */
2660 static int
2661 instr_alu_add_translate(struct rte_swx_pipeline *p,
2662                         struct action *action,
2663                         char **tokens,
2664                         int n_tokens,
2665                         struct instruction *instr,
2666                         struct instruction_data *data __rte_unused)
2667 {
2668         char *dst = tokens[1], *src = tokens[2];
2669         struct field *fdst, *fsrc;
2670         uint64_t src_val;
2671         uint32_t dst_struct_id = 0, src_struct_id = 0;
2672
2673         CHECK(n_tokens == 3, EINVAL);
2674
2675         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2676         CHECK(fdst, EINVAL);
2677         CHECK(!fdst->var_size, EINVAL);
2678
2679         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2680         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2681         if (fsrc) {
2682                 CHECK(!fsrc->var_size, EINVAL);
2683
2684                 instr->type = INSTR_ALU_ADD;
2685                 if (dst[0] == 'h' && src[0] != 'h')
2686                         instr->type = INSTR_ALU_ADD_HM;
2687                 if (dst[0] != 'h' && src[0] == 'h')
2688                         instr->type = INSTR_ALU_ADD_MH;
2689                 if (dst[0] == 'h' && src[0] == 'h')
2690                         instr->type = INSTR_ALU_ADD_HH;
2691
2692                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2693                 instr->alu.dst.n_bits = fdst->n_bits;
2694                 instr->alu.dst.offset = fdst->offset / 8;
2695                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2696                 instr->alu.src.n_bits = fsrc->n_bits;
2697                 instr->alu.src.offset = fsrc->offset / 8;
2698                 return 0;
2699         }
2700
2701         /* ADD_MI, ADD_HI. */
2702         src_val = strtoull(src, &src, 0);
2703         CHECK(!src[0], EINVAL);
2704
2705         instr->type = INSTR_ALU_ADD_MI;
2706         if (dst[0] == 'h')
2707                 instr->type = INSTR_ALU_ADD_HI;
2708
2709         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2710         instr->alu.dst.n_bits = fdst->n_bits;
2711         instr->alu.dst.offset = fdst->offset / 8;
2712         instr->alu.src_val = src_val;
2713         return 0;
2714 }
2715
2716 static int
2717 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2718                         struct action *action,
2719                         char **tokens,
2720                         int n_tokens,
2721                         struct instruction *instr,
2722                         struct instruction_data *data __rte_unused)
2723 {
2724         char *dst = tokens[1], *src = tokens[2];
2725         struct field *fdst, *fsrc;
2726         uint64_t src_val;
2727         uint32_t dst_struct_id = 0, src_struct_id = 0;
2728
2729         CHECK(n_tokens == 3, EINVAL);
2730
2731         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2732         CHECK(fdst, EINVAL);
2733         CHECK(!fdst->var_size, EINVAL);
2734
2735         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2736         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2737         if (fsrc) {
2738                 CHECK(!fsrc->var_size, EINVAL);
2739
2740                 instr->type = INSTR_ALU_SUB;
2741                 if (dst[0] == 'h' && src[0] != 'h')
2742                         instr->type = INSTR_ALU_SUB_HM;
2743                 if (dst[0] != 'h' && src[0] == 'h')
2744                         instr->type = INSTR_ALU_SUB_MH;
2745                 if (dst[0] == 'h' && src[0] == 'h')
2746                         instr->type = INSTR_ALU_SUB_HH;
2747
2748                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2749                 instr->alu.dst.n_bits = fdst->n_bits;
2750                 instr->alu.dst.offset = fdst->offset / 8;
2751                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2752                 instr->alu.src.n_bits = fsrc->n_bits;
2753                 instr->alu.src.offset = fsrc->offset / 8;
2754                 return 0;
2755         }
2756
2757         /* SUB_MI, SUB_HI. */
2758         src_val = strtoull(src, &src, 0);
2759         CHECK(!src[0], EINVAL);
2760
2761         instr->type = INSTR_ALU_SUB_MI;
2762         if (dst[0] == 'h')
2763                 instr->type = INSTR_ALU_SUB_HI;
2764
2765         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2766         instr->alu.dst.n_bits = fdst->n_bits;
2767         instr->alu.dst.offset = fdst->offset / 8;
2768         instr->alu.src_val = src_val;
2769         return 0;
2770 }
2771
2772 static int
2773 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2774                           struct action *action __rte_unused,
2775                           char **tokens,
2776                           int n_tokens,
2777                           struct instruction *instr,
2778                           struct instruction_data *data __rte_unused)
2779 {
2780         char *dst = tokens[1], *src = tokens[2];
2781         struct header *hdst, *hsrc;
2782         struct field *fdst, *fsrc;
2783
2784         CHECK(n_tokens == 3, EINVAL);
2785
2786         fdst = header_field_parse(p, dst, &hdst);
2787         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2788         CHECK(!fdst->var_size, EINVAL);
2789
2790         /* CKADD_FIELD. */
2791         fsrc = header_field_parse(p, src, &hsrc);
2792         if (fsrc) {
2793                 CHECK(!fsrc->var_size, EINVAL);
2794
2795                 instr->type = INSTR_ALU_CKADD_FIELD;
2796                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2797                 instr->alu.dst.n_bits = fdst->n_bits;
2798                 instr->alu.dst.offset = fdst->offset / 8;
2799                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2800                 instr->alu.src.n_bits = fsrc->n_bits;
2801                 instr->alu.src.offset = fsrc->offset / 8;
2802                 return 0;
2803         }
2804
2805         /* CKADD_STRUCT, CKADD_STRUCT20. */
2806         hsrc = header_parse(p, src);
2807         CHECK(hsrc, EINVAL);
2808         CHECK(!hsrc->st->var_size, EINVAL);
2809
2810         instr->type = INSTR_ALU_CKADD_STRUCT;
2811         if ((hsrc->st->n_bits / 8) == 20)
2812                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2813
2814         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2815         instr->alu.dst.n_bits = fdst->n_bits;
2816         instr->alu.dst.offset = fdst->offset / 8;
2817         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2818         instr->alu.src.n_bits = hsrc->st->n_bits;
2819         instr->alu.src.offset = 0; /* Unused. */
2820         return 0;
2821 }
2822
2823 static int
2824 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2825                           struct action *action __rte_unused,
2826                           char **tokens,
2827                           int n_tokens,
2828                           struct instruction *instr,
2829                           struct instruction_data *data __rte_unused)
2830 {
2831         char *dst = tokens[1], *src = tokens[2];
2832         struct header *hdst, *hsrc;
2833         struct field *fdst, *fsrc;
2834
2835         CHECK(n_tokens == 3, EINVAL);
2836
2837         fdst = header_field_parse(p, dst, &hdst);
2838         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2839         CHECK(!fdst->var_size, EINVAL);
2840
2841         fsrc = header_field_parse(p, src, &hsrc);
2842         CHECK(fsrc, EINVAL);
2843         CHECK(!fsrc->var_size, EINVAL);
2844
2845         instr->type = INSTR_ALU_CKSUB_FIELD;
2846         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2847         instr->alu.dst.n_bits = fdst->n_bits;
2848         instr->alu.dst.offset = fdst->offset / 8;
2849         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2850         instr->alu.src.n_bits = fsrc->n_bits;
2851         instr->alu.src.offset = fsrc->offset / 8;
2852         return 0;
2853 }
2854
2855 static int
2856 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2857                         struct action *action,
2858                         char **tokens,
2859                         int n_tokens,
2860                         struct instruction *instr,
2861                         struct instruction_data *data __rte_unused)
2862 {
2863         char *dst = tokens[1], *src = tokens[2];
2864         struct field *fdst, *fsrc;
2865         uint64_t src_val;
2866         uint32_t dst_struct_id = 0, src_struct_id = 0;
2867
2868         CHECK(n_tokens == 3, EINVAL);
2869
2870         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2871         CHECK(fdst, EINVAL);
2872         CHECK(!fdst->var_size, EINVAL);
2873
2874         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2875         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2876         if (fsrc) {
2877                 CHECK(!fsrc->var_size, EINVAL);
2878
2879                 instr->type = INSTR_ALU_SHL;
2880                 if (dst[0] == 'h' && src[0] != 'h')
2881                         instr->type = INSTR_ALU_SHL_HM;
2882                 if (dst[0] != 'h' && src[0] == 'h')
2883                         instr->type = INSTR_ALU_SHL_MH;
2884                 if (dst[0] == 'h' && src[0] == 'h')
2885                         instr->type = INSTR_ALU_SHL_HH;
2886
2887                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2888                 instr->alu.dst.n_bits = fdst->n_bits;
2889                 instr->alu.dst.offset = fdst->offset / 8;
2890                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2891                 instr->alu.src.n_bits = fsrc->n_bits;
2892                 instr->alu.src.offset = fsrc->offset / 8;
2893                 return 0;
2894         }
2895
2896         /* SHL_MI, SHL_HI. */
2897         src_val = strtoull(src, &src, 0);
2898         CHECK(!src[0], EINVAL);
2899
2900         instr->type = INSTR_ALU_SHL_MI;
2901         if (dst[0] == 'h')
2902                 instr->type = INSTR_ALU_SHL_HI;
2903
2904         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2905         instr->alu.dst.n_bits = fdst->n_bits;
2906         instr->alu.dst.offset = fdst->offset / 8;
2907         instr->alu.src_val = src_val;
2908         return 0;
2909 }
2910
2911 static int
2912 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2913                         struct action *action,
2914                         char **tokens,
2915                         int n_tokens,
2916                         struct instruction *instr,
2917                         struct instruction_data *data __rte_unused)
2918 {
2919         char *dst = tokens[1], *src = tokens[2];
2920         struct field *fdst, *fsrc;
2921         uint64_t src_val;
2922         uint32_t dst_struct_id = 0, src_struct_id = 0;
2923
2924         CHECK(n_tokens == 3, EINVAL);
2925
2926         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2927         CHECK(fdst, EINVAL);
2928         CHECK(!fdst->var_size, EINVAL);
2929
2930         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2931         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2932         if (fsrc) {
2933                 CHECK(!fsrc->var_size, EINVAL);
2934
2935                 instr->type = INSTR_ALU_SHR;
2936                 if (dst[0] == 'h' && src[0] != 'h')
2937                         instr->type = INSTR_ALU_SHR_HM;
2938                 if (dst[0] != 'h' && src[0] == 'h')
2939                         instr->type = INSTR_ALU_SHR_MH;
2940                 if (dst[0] == 'h' && src[0] == 'h')
2941                         instr->type = INSTR_ALU_SHR_HH;
2942
2943                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2944                 instr->alu.dst.n_bits = fdst->n_bits;
2945                 instr->alu.dst.offset = fdst->offset / 8;
2946                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2947                 instr->alu.src.n_bits = fsrc->n_bits;
2948                 instr->alu.src.offset = fsrc->offset / 8;
2949                 return 0;
2950         }
2951
2952         /* SHR_MI, SHR_HI. */
2953         src_val = strtoull(src, &src, 0);
2954         CHECK(!src[0], EINVAL);
2955
2956         instr->type = INSTR_ALU_SHR_MI;
2957         if (dst[0] == 'h')
2958                 instr->type = INSTR_ALU_SHR_HI;
2959
2960         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2961         instr->alu.dst.n_bits = fdst->n_bits;
2962         instr->alu.dst.offset = fdst->offset / 8;
2963         instr->alu.src_val = src_val;
2964         return 0;
2965 }
2966
2967 static int
2968 instr_alu_and_translate(struct rte_swx_pipeline *p,
2969                         struct action *action,
2970                         char **tokens,
2971                         int n_tokens,
2972                         struct instruction *instr,
2973                         struct instruction_data *data __rte_unused)
2974 {
2975         char *dst = tokens[1], *src = tokens[2];
2976         struct field *fdst, *fsrc;
2977         uint64_t src_val;
2978         uint32_t dst_struct_id = 0, src_struct_id = 0;
2979
2980         CHECK(n_tokens == 3, EINVAL);
2981
2982         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2983         CHECK(fdst, EINVAL);
2984         CHECK(!fdst->var_size, EINVAL);
2985
2986         /* AND, AND_MH, AND_HM, AND_HH. */
2987         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2988         if (fsrc) {
2989                 CHECK(!fsrc->var_size, EINVAL);
2990
2991                 instr->type = INSTR_ALU_AND;
2992                 if (dst[0] != 'h' && src[0] == 'h')
2993                         instr->type = INSTR_ALU_AND_MH;
2994                 if (dst[0] == 'h' && src[0] != 'h')
2995                         instr->type = INSTR_ALU_AND_HM;
2996                 if (dst[0] == 'h' && src[0] == 'h')
2997                         instr->type = INSTR_ALU_AND_HH;
2998
2999                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3000                 instr->alu.dst.n_bits = fdst->n_bits;
3001                 instr->alu.dst.offset = fdst->offset / 8;
3002                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3003                 instr->alu.src.n_bits = fsrc->n_bits;
3004                 instr->alu.src.offset = fsrc->offset / 8;
3005                 return 0;
3006         }
3007
3008         /* AND_I. */
3009         src_val = strtoull(src, &src, 0);
3010         CHECK(!src[0], EINVAL);
3011
3012         if (dst[0] == 'h')
3013                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3014
3015         instr->type = INSTR_ALU_AND_I;
3016         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3017         instr->alu.dst.n_bits = fdst->n_bits;
3018         instr->alu.dst.offset = fdst->offset / 8;
3019         instr->alu.src_val = src_val;
3020         return 0;
3021 }
3022
3023 static int
3024 instr_alu_or_translate(struct rte_swx_pipeline *p,
3025                        struct action *action,
3026                        char **tokens,
3027                        int n_tokens,
3028                        struct instruction *instr,
3029                        struct instruction_data *data __rte_unused)
3030 {
3031         char *dst = tokens[1], *src = tokens[2];
3032         struct field *fdst, *fsrc;
3033         uint64_t src_val;
3034         uint32_t dst_struct_id = 0, src_struct_id = 0;
3035
3036         CHECK(n_tokens == 3, EINVAL);
3037
3038         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3039         CHECK(fdst, EINVAL);
3040         CHECK(!fdst->var_size, EINVAL);
3041
3042         /* OR, OR_MH, OR_HM, OR_HH. */
3043         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3044         if (fsrc) {
3045                 CHECK(!fsrc->var_size, EINVAL);
3046
3047                 instr->type = INSTR_ALU_OR;
3048                 if (dst[0] != 'h' && src[0] == 'h')
3049                         instr->type = INSTR_ALU_OR_MH;
3050                 if (dst[0] == 'h' && src[0] != 'h')
3051                         instr->type = INSTR_ALU_OR_HM;
3052                 if (dst[0] == 'h' && src[0] == 'h')
3053                         instr->type = INSTR_ALU_OR_HH;
3054
3055                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3056                 instr->alu.dst.n_bits = fdst->n_bits;
3057                 instr->alu.dst.offset = fdst->offset / 8;
3058                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3059                 instr->alu.src.n_bits = fsrc->n_bits;
3060                 instr->alu.src.offset = fsrc->offset / 8;
3061                 return 0;
3062         }
3063
3064         /* OR_I. */
3065         src_val = strtoull(src, &src, 0);
3066         CHECK(!src[0], EINVAL);
3067
3068         if (dst[0] == 'h')
3069                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3070
3071         instr->type = INSTR_ALU_OR_I;
3072         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3073         instr->alu.dst.n_bits = fdst->n_bits;
3074         instr->alu.dst.offset = fdst->offset / 8;
3075         instr->alu.src_val = src_val;
3076         return 0;
3077 }
3078
3079 static int
3080 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3081                         struct action *action,
3082                         char **tokens,
3083                         int n_tokens,
3084                         struct instruction *instr,
3085                         struct instruction_data *data __rte_unused)
3086 {
3087         char *dst = tokens[1], *src = tokens[2];
3088         struct field *fdst, *fsrc;
3089         uint64_t src_val;
3090         uint32_t dst_struct_id = 0, src_struct_id = 0;
3091
3092         CHECK(n_tokens == 3, EINVAL);
3093
3094         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3095         CHECK(fdst, EINVAL);
3096         CHECK(!fdst->var_size, EINVAL);
3097
3098         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3099         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3100         if (fsrc) {
3101                 CHECK(!fsrc->var_size, EINVAL);
3102
3103                 instr->type = INSTR_ALU_XOR;
3104                 if (dst[0] != 'h' && src[0] == 'h')
3105                         instr->type = INSTR_ALU_XOR_MH;
3106                 if (dst[0] == 'h' && src[0] != 'h')
3107                         instr->type = INSTR_ALU_XOR_HM;
3108                 if (dst[0] == 'h' && src[0] == 'h')
3109                         instr->type = INSTR_ALU_XOR_HH;
3110
3111                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3112                 instr->alu.dst.n_bits = fdst->n_bits;
3113                 instr->alu.dst.offset = fdst->offset / 8;
3114                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3115                 instr->alu.src.n_bits = fsrc->n_bits;
3116                 instr->alu.src.offset = fsrc->offset / 8;
3117                 return 0;
3118         }
3119
3120         /* XOR_I. */
3121         src_val = strtoull(src, &src, 0);
3122         CHECK(!src[0], EINVAL);
3123
3124         if (dst[0] == 'h')
3125                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3126
3127         instr->type = INSTR_ALU_XOR_I;
3128         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3129         instr->alu.dst.n_bits = fdst->n_bits;
3130         instr->alu.dst.offset = fdst->offset / 8;
3131         instr->alu.src_val = src_val;
3132         return 0;
3133 }
3134
3135 static inline void
3136 instr_alu_add_exec(struct rte_swx_pipeline *p)
3137 {
3138         struct thread *t = &p->threads[p->thread_id];
3139         struct instruction *ip = t->ip;
3140
3141         TRACE("[Thread %2u] add\n", p->thread_id);
3142
3143         /* Structs. */
3144         ALU(t, ip, +);
3145
3146         /* Thread. */
3147         thread_ip_inc(p);
3148 }
3149
3150 static inline void
3151 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3152 {
3153         struct thread *t = &p->threads[p->thread_id];
3154         struct instruction *ip = t->ip;
3155
3156         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3157
3158         /* Structs. */
3159         ALU_MH(t, ip, +);
3160
3161         /* Thread. */
3162         thread_ip_inc(p);
3163 }
3164
3165 static inline void
3166 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3167 {
3168         struct thread *t = &p->threads[p->thread_id];
3169         struct instruction *ip = t->ip;
3170
3171         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3172
3173         /* Structs. */
3174         ALU_HM(t, ip, +);
3175
3176         /* Thread. */
3177         thread_ip_inc(p);
3178 }
3179
3180 static inline void
3181 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3182 {
3183         struct thread *t = &p->threads[p->thread_id];
3184         struct instruction *ip = t->ip;
3185
3186         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3187
3188         /* Structs. */
3189         ALU_HH(t, ip, +);
3190
3191         /* Thread. */
3192         thread_ip_inc(p);
3193 }
3194
3195 static inline void
3196 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3197 {
3198         struct thread *t = &p->threads[p->thread_id];
3199         struct instruction *ip = t->ip;
3200
3201         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3202
3203         /* Structs. */
3204         ALU_MI(t, ip, +);
3205
3206         /* Thread. */
3207         thread_ip_inc(p);
3208 }
3209
3210 static inline void
3211 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3212 {
3213         struct thread *t = &p->threads[p->thread_id];
3214         struct instruction *ip = t->ip;
3215
3216         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3217
3218         /* Structs. */
3219         ALU_HI(t, ip, +);
3220
3221         /* Thread. */
3222         thread_ip_inc(p);
3223 }
3224
3225 static inline void
3226 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3227 {
3228         struct thread *t = &p->threads[p->thread_id];
3229         struct instruction *ip = t->ip;
3230
3231         TRACE("[Thread %2u] sub\n", p->thread_id);
3232
3233         /* Structs. */
3234         ALU(t, ip, -);
3235
3236         /* Thread. */
3237         thread_ip_inc(p);
3238 }
3239
3240 static inline void
3241 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3242 {
3243         struct thread *t = &p->threads[p->thread_id];
3244         struct instruction *ip = t->ip;
3245
3246         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3247
3248         /* Structs. */
3249         ALU_MH(t, ip, -);
3250
3251         /* Thread. */
3252         thread_ip_inc(p);
3253 }
3254
3255 static inline void
3256 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3257 {
3258         struct thread *t = &p->threads[p->thread_id];
3259         struct instruction *ip = t->ip;
3260
3261         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3262
3263         /* Structs. */
3264         ALU_HM(t, ip, -);
3265
3266         /* Thread. */
3267         thread_ip_inc(p);
3268 }
3269
3270 static inline void
3271 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3272 {
3273         struct thread *t = &p->threads[p->thread_id];
3274         struct instruction *ip = t->ip;
3275
3276         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3277
3278         /* Structs. */
3279         ALU_HH(t, ip, -);
3280
3281         /* Thread. */
3282         thread_ip_inc(p);
3283 }
3284
3285 static inline void
3286 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3287 {
3288         struct thread *t = &p->threads[p->thread_id];
3289         struct instruction *ip = t->ip;
3290
3291         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3292
3293         /* Structs. */
3294         ALU_MI(t, ip, -);
3295
3296         /* Thread. */
3297         thread_ip_inc(p);
3298 }
3299
3300 static inline void
3301 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3302 {
3303         struct thread *t = &p->threads[p->thread_id];
3304         struct instruction *ip = t->ip;
3305
3306         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3307
3308         /* Structs. */
3309         ALU_HI(t, ip, -);
3310
3311         /* Thread. */
3312         thread_ip_inc(p);
3313 }
3314
3315 static inline void
3316 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3317 {
3318         struct thread *t = &p->threads[p->thread_id];
3319         struct instruction *ip = t->ip;
3320
3321         TRACE("[Thread %2u] shl\n", p->thread_id);
3322
3323         /* Structs. */
3324         ALU(t, ip, <<);
3325
3326         /* Thread. */
3327         thread_ip_inc(p);
3328 }
3329
3330 static inline void
3331 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3332 {
3333         struct thread *t = &p->threads[p->thread_id];
3334         struct instruction *ip = t->ip;
3335
3336         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3337
3338         /* Structs. */
3339         ALU_MH(t, ip, <<);
3340
3341         /* Thread. */
3342         thread_ip_inc(p);
3343 }
3344
3345 static inline void
3346 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3347 {
3348         struct thread *t = &p->threads[p->thread_id];
3349         struct instruction *ip = t->ip;
3350
3351         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3352
3353         /* Structs. */
3354         ALU_HM(t, ip, <<);
3355
3356         /* Thread. */
3357         thread_ip_inc(p);
3358 }
3359
3360 static inline void
3361 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3362 {
3363         struct thread *t = &p->threads[p->thread_id];
3364         struct instruction *ip = t->ip;
3365
3366         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3367
3368         /* Structs. */
3369         ALU_HH(t, ip, <<);
3370
3371         /* Thread. */
3372         thread_ip_inc(p);
3373 }
3374
3375 static inline void
3376 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3377 {
3378         struct thread *t = &p->threads[p->thread_id];
3379         struct instruction *ip = t->ip;
3380
3381         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3382
3383         /* Structs. */
3384         ALU_MI(t, ip, <<);
3385
3386         /* Thread. */
3387         thread_ip_inc(p);
3388 }
3389
3390 static inline void
3391 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3392 {
3393         struct thread *t = &p->threads[p->thread_id];
3394         struct instruction *ip = t->ip;
3395
3396         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3397
3398         /* Structs. */
3399         ALU_HI(t, ip, <<);
3400
3401         /* Thread. */
3402         thread_ip_inc(p);
3403 }
3404
3405 static inline void
3406 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3407 {
3408         struct thread *t = &p->threads[p->thread_id];
3409         struct instruction *ip = t->ip;
3410
3411         TRACE("[Thread %2u] shr\n", p->thread_id);
3412
3413         /* Structs. */
3414         ALU(t, ip, >>);
3415
3416         /* Thread. */
3417         thread_ip_inc(p);
3418 }
3419
3420 static inline void
3421 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3422 {
3423         struct thread *t = &p->threads[p->thread_id];
3424         struct instruction *ip = t->ip;
3425
3426         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3427
3428         /* Structs. */
3429         ALU_MH(t, ip, >>);
3430
3431         /* Thread. */
3432         thread_ip_inc(p);
3433 }
3434
3435 static inline void
3436 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3437 {
3438         struct thread *t = &p->threads[p->thread_id];
3439         struct instruction *ip = t->ip;
3440
3441         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3442
3443         /* Structs. */
3444         ALU_HM(t, ip, >>);
3445
3446         /* Thread. */
3447         thread_ip_inc(p);
3448 }
3449
3450 static inline void
3451 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455
3456         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3457
3458         /* Structs. */
3459         ALU_HH(t, ip, >>);
3460
3461         /* Thread. */
3462         thread_ip_inc(p);
3463 }
3464
3465 static inline void
3466 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3467 {
3468         struct thread *t = &p->threads[p->thread_id];
3469         struct instruction *ip = t->ip;
3470
3471         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3472
3473         /* Structs. */
3474         ALU_MI(t, ip, >>);
3475
3476         /* Thread. */
3477         thread_ip_inc(p);
3478 }
3479
3480 static inline void
3481 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3482 {
3483         struct thread *t = &p->threads[p->thread_id];
3484         struct instruction *ip = t->ip;
3485
3486         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3487
3488         /* Structs. */
3489         ALU_HI(t, ip, >>);
3490
3491         /* Thread. */
3492         thread_ip_inc(p);
3493 }
3494
3495 static inline void
3496 instr_alu_and_exec(struct rte_swx_pipeline *p)
3497 {
3498         struct thread *t = &p->threads[p->thread_id];
3499         struct instruction *ip = t->ip;
3500
3501         TRACE("[Thread %2u] and\n", p->thread_id);
3502
3503         /* Structs. */
3504         ALU(t, ip, &);
3505
3506         /* Thread. */
3507         thread_ip_inc(p);
3508 }
3509
3510 static inline void
3511 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3512 {
3513         struct thread *t = &p->threads[p->thread_id];
3514         struct instruction *ip = t->ip;
3515
3516         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3517
3518         /* Structs. */
3519         ALU_MH(t, ip, &);
3520
3521         /* Thread. */
3522         thread_ip_inc(p);
3523 }
3524
3525 static inline void
3526 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3527 {
3528         struct thread *t = &p->threads[p->thread_id];
3529         struct instruction *ip = t->ip;
3530
3531         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3532
3533         /* Structs. */
3534         ALU_HM_FAST(t, ip, &);
3535
3536         /* Thread. */
3537         thread_ip_inc(p);
3538 }
3539
3540 static inline void
3541 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3542 {
3543         struct thread *t = &p->threads[p->thread_id];
3544         struct instruction *ip = t->ip;
3545
3546         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3547
3548         /* Structs. */
3549         ALU_HH_FAST(t, ip, &);
3550
3551         /* Thread. */
3552         thread_ip_inc(p);
3553 }
3554
3555 static inline void
3556 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3557 {
3558         struct thread *t = &p->threads[p->thread_id];
3559         struct instruction *ip = t->ip;
3560
3561         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3562
3563         /* Structs. */
3564         ALU_I(t, ip, &);
3565
3566         /* Thread. */
3567         thread_ip_inc(p);
3568 }
3569
3570 static inline void
3571 instr_alu_or_exec(struct rte_swx_pipeline *p)
3572 {
3573         struct thread *t = &p->threads[p->thread_id];
3574         struct instruction *ip = t->ip;
3575
3576         TRACE("[Thread %2u] or\n", p->thread_id);
3577
3578         /* Structs. */
3579         ALU(t, ip, |);
3580
3581         /* Thread. */
3582         thread_ip_inc(p);
3583 }
3584
3585 static inline void
3586 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3587 {
3588         struct thread *t = &p->threads[p->thread_id];
3589         struct instruction *ip = t->ip;
3590
3591         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3592
3593         /* Structs. */
3594         ALU_MH(t, ip, |);
3595
3596         /* Thread. */
3597         thread_ip_inc(p);
3598 }
3599
3600 static inline void
3601 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3602 {
3603         struct thread *t = &p->threads[p->thread_id];
3604         struct instruction *ip = t->ip;
3605
3606         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3607
3608         /* Structs. */
3609         ALU_HM_FAST(t, ip, |);
3610
3611         /* Thread. */
3612         thread_ip_inc(p);
3613 }
3614
3615 static inline void
3616 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3617 {
3618         struct thread *t = &p->threads[p->thread_id];
3619         struct instruction *ip = t->ip;
3620
3621         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3622
3623         /* Structs. */
3624         ALU_HH_FAST(t, ip, |);
3625
3626         /* Thread. */
3627         thread_ip_inc(p);
3628 }
3629
3630 static inline void
3631 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3632 {
3633         struct thread *t = &p->threads[p->thread_id];
3634         struct instruction *ip = t->ip;
3635
3636         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3637
3638         /* Structs. */
3639         ALU_I(t, ip, |);
3640
3641         /* Thread. */
3642         thread_ip_inc(p);
3643 }
3644
3645 static inline void
3646 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3647 {
3648         struct thread *t = &p->threads[p->thread_id];
3649         struct instruction *ip = t->ip;
3650
3651         TRACE("[Thread %2u] xor\n", p->thread_id);
3652
3653         /* Structs. */
3654         ALU(t, ip, ^);
3655
3656         /* Thread. */
3657         thread_ip_inc(p);
3658 }
3659
3660 static inline void
3661 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3662 {
3663         struct thread *t = &p->threads[p->thread_id];
3664         struct instruction *ip = t->ip;
3665
3666         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3667
3668         /* Structs. */
3669         ALU_MH(t, ip, ^);
3670
3671         /* Thread. */
3672         thread_ip_inc(p);
3673 }
3674
3675 static inline void
3676 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3677 {
3678         struct thread *t = &p->threads[p->thread_id];
3679         struct instruction *ip = t->ip;
3680
3681         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3682
3683         /* Structs. */
3684         ALU_HM_FAST(t, ip, ^);
3685
3686         /* Thread. */
3687         thread_ip_inc(p);
3688 }
3689
3690 static inline void
3691 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3692 {
3693         struct thread *t = &p->threads[p->thread_id];
3694         struct instruction *ip = t->ip;
3695
3696         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3697
3698         /* Structs. */
3699         ALU_HH_FAST(t, ip, ^);
3700
3701         /* Thread. */
3702         thread_ip_inc(p);
3703 }
3704
3705 static inline void
3706 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3707 {
3708         struct thread *t = &p->threads[p->thread_id];
3709         struct instruction *ip = t->ip;
3710
3711         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3712
3713         /* Structs. */
3714         ALU_I(t, ip, ^);
3715
3716         /* Thread. */
3717         thread_ip_inc(p);
3718 }
3719
3720 static inline void
3721 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3722 {
3723         struct thread *t = &p->threads[p->thread_id];
3724         struct instruction *ip = t->ip;
3725         uint8_t *dst_struct, *src_struct;
3726         uint16_t *dst16_ptr, dst;
3727         uint64_t *src64_ptr, src64, src64_mask, src;
3728         uint64_t r;
3729
3730         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3731
3732         /* Structs. */
3733         dst_struct = t->structs[ip->alu.dst.struct_id];
3734         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3735         dst = *dst16_ptr;
3736
3737         src_struct = t->structs[ip->alu.src.struct_id];
3738         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3739         src64 = *src64_ptr;
3740         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3741         src = src64 & src64_mask;
3742
3743         r = dst;
3744         r = ~r & 0xFFFF;
3745
3746         /* The first input (r) is a 16-bit number. The second and the third
3747          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3748          * three numbers (output r) is a 34-bit number.
3749          */
3750         r += (src >> 32) + (src & 0xFFFFFFFF);
3751
3752         /* The first input is a 16-bit number. The second input is an 18-bit
3753          * number. In the worst case scenario, the sum of the two numbers is a
3754          * 19-bit number.
3755          */
3756         r = (r & 0xFFFF) + (r >> 16);
3757
3758         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3759          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3760          */
3761         r = (r & 0xFFFF) + (r >> 16);
3762
3763         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3764          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3765          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3766          * therefore the output r is always a 16-bit number.
3767          */
3768         r = (r & 0xFFFF) + (r >> 16);
3769
3770         r = ~r & 0xFFFF;
3771         r = r ? r : 0xFFFF;
3772
3773         *dst16_ptr = (uint16_t)r;
3774
3775         /* Thread. */
3776         thread_ip_inc(p);
3777 }
3778
3779 static inline void
3780 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3781 {
3782         struct thread *t = &p->threads[p->thread_id];
3783         struct instruction *ip = t->ip;
3784         uint8_t *dst_struct, *src_struct;
3785         uint16_t *dst16_ptr, dst;
3786         uint64_t *src64_ptr, src64, src64_mask, src;
3787         uint64_t r;
3788
3789         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3790
3791         /* Structs. */
3792         dst_struct = t->structs[ip->alu.dst.struct_id];
3793         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3794         dst = *dst16_ptr;
3795
3796         src_struct = t->structs[ip->alu.src.struct_id];
3797         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3798         src64 = *src64_ptr;
3799         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3800         src = src64 & src64_mask;
3801
3802         r = dst;
3803         r = ~r & 0xFFFF;
3804
3805         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3806          * the following sequence of operations in 2's complement arithmetic:
3807          *    a '- b = (a - b) % 0xFFFF.
3808          *
3809          * In order to prevent an underflow for the below subtraction, in which
3810          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3811          * minuend), we first add a multiple of the 0xFFFF modulus to the
3812          * minuend. The number we add to the minuend needs to be a 34-bit number
3813          * or higher, so for readability reasons we picked the 36-bit multiple.
3814          * We are effectively turning the 16-bit minuend into a 36-bit number:
3815          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3816          */
3817         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3818
3819         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3820          * result (the output r) is a 36-bit number.
3821          */
3822         r -= (src >> 32) + (src & 0xFFFFFFFF);
3823
3824         /* The first input is a 16-bit number. The second input is a 20-bit
3825          * number. Their sum is a 21-bit number.
3826          */
3827         r = (r & 0xFFFF) + (r >> 16);
3828
3829         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3830          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3831          */
3832         r = (r & 0xFFFF) + (r >> 16);
3833
3834         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3835          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3836          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3837          * generated, therefore the output r is always a 16-bit number.
3838          */
3839         r = (r & 0xFFFF) + (r >> 16);
3840
3841         r = ~r & 0xFFFF;
3842         r = r ? r : 0xFFFF;
3843
3844         *dst16_ptr = (uint16_t)r;
3845
3846         /* Thread. */
3847         thread_ip_inc(p);
3848 }
3849
3850 static inline void
3851 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3852 {
3853         struct thread *t = &p->threads[p->thread_id];
3854         struct instruction *ip = t->ip;
3855         uint8_t *dst_struct, *src_struct;
3856         uint16_t *dst16_ptr;
3857         uint32_t *src32_ptr;
3858         uint64_t r0, r1;
3859
3860         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3861
3862         /* Structs. */
3863         dst_struct = t->structs[ip->alu.dst.struct_id];
3864         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3865
3866         src_struct = t->structs[ip->alu.src.struct_id];
3867         src32_ptr = (uint32_t *)&src_struct[0];
3868
3869         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3870         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3871         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3872         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3873         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3874
3875         /* The first input is a 16-bit number. The second input is a 19-bit
3876          * number. Their sum is a 20-bit number.
3877          */
3878         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3879
3880         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3881          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3882          */
3883         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3884
3885         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3886          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3887          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3888          * generated, therefore the output r is always a 16-bit number.
3889          */
3890         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3891
3892         r0 = ~r0 & 0xFFFF;
3893         r0 = r0 ? r0 : 0xFFFF;
3894
3895         *dst16_ptr = (uint16_t)r0;
3896
3897         /* Thread. */
3898         thread_ip_inc(p);
3899 }
3900
3901 static inline void
3902 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3903 {
3904         struct thread *t = &p->threads[p->thread_id];
3905         struct instruction *ip = t->ip;
3906         uint8_t *dst_struct, *src_struct;
3907         uint16_t *dst16_ptr;
3908         uint32_t *src32_ptr;
3909         uint64_t r = 0;
3910         uint32_t i;
3911
3912         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3913
3914         /* Structs. */
3915         dst_struct = t->structs[ip->alu.dst.struct_id];
3916         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3917
3918         src_struct = t->structs[ip->alu.src.struct_id];
3919         src32_ptr = (uint32_t *)&src_struct[0];
3920
3921         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3922          * Therefore, in the worst case scenario, a 35-bit number is added to a
3923          * 16-bit number (the input r), so the output r is 36-bit number.
3924          */
3925         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3926                 r += *src32_ptr;
3927
3928         /* The first input is a 16-bit number. The second input is a 20-bit
3929          * number. Their sum is a 21-bit number.
3930          */
3931         r = (r & 0xFFFF) + (r >> 16);
3932
3933         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3934          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3935          */
3936         r = (r & 0xFFFF) + (r >> 16);
3937
3938         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3939          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3940          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3941          * generated, therefore the output r is always a 16-bit number.
3942          */
3943         r = (r & 0xFFFF) + (r >> 16);
3944
3945         r = ~r & 0xFFFF;
3946         r = r ? r : 0xFFFF;
3947
3948         *dst16_ptr = (uint16_t)r;
3949
3950         /* Thread. */
3951         thread_ip_inc(p);
3952 }
3953
3954 /*
3955  * Register array.
3956  */
3957 static struct regarray *
3958 regarray_find(struct rte_swx_pipeline *p, const char *name);
3959
3960 static int
3961 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3962                       struct action *action,
3963                       char **tokens,
3964                       int n_tokens,
3965                       struct instruction *instr,
3966                       struct instruction_data *data __rte_unused)
3967 {
3968         char *regarray = tokens[1], *idx = tokens[2];
3969         struct regarray *r;
3970         struct field *fidx;
3971         uint32_t idx_struct_id, idx_val;
3972
3973         CHECK(n_tokens == 3, EINVAL);
3974
3975         r = regarray_find(p, regarray);
3976         CHECK(r, EINVAL);
3977
3978         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3979         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3980         if (fidx) {
3981                 CHECK(!fidx->var_size, EINVAL);
3982
3983                 instr->type = INSTR_REGPREFETCH_RM;
3984                 if (idx[0] == 'h')
3985                         instr->type = INSTR_REGPREFETCH_RH;
3986
3987                 instr->regarray.regarray_id = r->id;
3988                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3989                 instr->regarray.idx.n_bits = fidx->n_bits;
3990                 instr->regarray.idx.offset = fidx->offset / 8;
3991                 instr->regarray.dstsrc_val = 0; /* Unused. */
3992                 return 0;
3993         }
3994
3995         /* REGPREFETCH_RI. */
3996         idx_val = strtoul(idx, &idx, 0);
3997         CHECK(!idx[0], EINVAL);
3998
3999         instr->type = INSTR_REGPREFETCH_RI;
4000         instr->regarray.regarray_id = r->id;
4001         instr->regarray.idx_val = idx_val;
4002         instr->regarray.dstsrc_val = 0; /* Unused. */
4003         return 0;
4004 }
4005
4006 static int
4007 instr_regrd_translate(struct rte_swx_pipeline *p,
4008                       struct action *action,
4009                       char **tokens,
4010                       int n_tokens,
4011                       struct instruction *instr,
4012                       struct instruction_data *data __rte_unused)
4013 {
4014         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4015         struct regarray *r;
4016         struct field *fdst, *fidx;
4017         uint32_t dst_struct_id, idx_struct_id, idx_val;
4018
4019         CHECK(n_tokens == 4, EINVAL);
4020
4021         r = regarray_find(p, regarray);
4022         CHECK(r, EINVAL);
4023
4024         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4025         CHECK(fdst, EINVAL);
4026         CHECK(!fdst->var_size, EINVAL);
4027
4028         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4029         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4030         if (fidx) {
4031                 CHECK(!fidx->var_size, EINVAL);
4032
4033                 instr->type = INSTR_REGRD_MRM;
4034                 if (dst[0] == 'h' && idx[0] != 'h')
4035                         instr->type = INSTR_REGRD_HRM;
4036                 if (dst[0] != 'h' && idx[0] == 'h')
4037                         instr->type = INSTR_REGRD_MRH;
4038                 if (dst[0] == 'h' && idx[0] == 'h')
4039                         instr->type = INSTR_REGRD_HRH;
4040
4041                 instr->regarray.regarray_id = r->id;
4042                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4043                 instr->regarray.idx.n_bits = fidx->n_bits;
4044                 instr->regarray.idx.offset = fidx->offset / 8;
4045                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4046                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4047                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4048                 return 0;
4049         }
4050
4051         /* REGRD_MRI, REGRD_HRI. */
4052         idx_val = strtoul(idx, &idx, 0);
4053         CHECK(!idx[0], EINVAL);
4054
4055         instr->type = INSTR_REGRD_MRI;
4056         if (dst[0] == 'h')
4057                 instr->type = INSTR_REGRD_HRI;
4058
4059         instr->regarray.regarray_id = r->id;
4060         instr->regarray.idx_val = idx_val;
4061         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4062         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4063         instr->regarray.dstsrc.offset = fdst->offset / 8;
4064         return 0;
4065 }
4066
4067 static int
4068 instr_regwr_translate(struct rte_swx_pipeline *p,
4069                       struct action *action,
4070                       char **tokens,
4071                       int n_tokens,
4072                       struct instruction *instr,
4073                       struct instruction_data *data __rte_unused)
4074 {
4075         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4076         struct regarray *r;
4077         struct field *fidx, *fsrc;
4078         uint64_t src_val;
4079         uint32_t idx_struct_id, idx_val, src_struct_id;
4080
4081         CHECK(n_tokens == 4, EINVAL);
4082
4083         r = regarray_find(p, regarray);
4084         CHECK(r, EINVAL);
4085
4086         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4087         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4088         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4089         if (fidx && fsrc) {
4090                 CHECK(!fidx->var_size, EINVAL);
4091                 CHECK(!fsrc->var_size, EINVAL);
4092
4093                 instr->type = INSTR_REGWR_RMM;
4094                 if (idx[0] == 'h' && src[0] != 'h')
4095                         instr->type = INSTR_REGWR_RHM;
4096                 if (idx[0] != 'h' && src[0] == 'h')
4097                         instr->type = INSTR_REGWR_RMH;
4098                 if (idx[0] == 'h' && src[0] == 'h')
4099                         instr->type = INSTR_REGWR_RHH;
4100
4101                 instr->regarray.regarray_id = r->id;
4102                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4103                 instr->regarray.idx.n_bits = fidx->n_bits;
4104                 instr->regarray.idx.offset = fidx->offset / 8;
4105                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4106                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4107                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4108                 return 0;
4109         }
4110
4111         /* REGWR_RHI, REGWR_RMI. */
4112         if (fidx && !fsrc) {
4113                 CHECK(!fidx->var_size, EINVAL);
4114
4115                 src_val = strtoull(src, &src, 0);
4116                 CHECK(!src[0], EINVAL);
4117
4118                 instr->type = INSTR_REGWR_RMI;
4119                 if (idx[0] == 'h')
4120                         instr->type = INSTR_REGWR_RHI;
4121
4122                 instr->regarray.regarray_id = r->id;
4123                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4124                 instr->regarray.idx.n_bits = fidx->n_bits;
4125                 instr->regarray.idx.offset = fidx->offset / 8;
4126                 instr->regarray.dstsrc_val = src_val;
4127                 return 0;
4128         }
4129
4130         /* REGWR_RIH, REGWR_RIM. */
4131         if (!fidx && fsrc) {
4132                 idx_val = strtoul(idx, &idx, 0);
4133                 CHECK(!idx[0], EINVAL);
4134
4135                 CHECK(!fsrc->var_size, EINVAL);
4136
4137                 instr->type = INSTR_REGWR_RIM;
4138                 if (src[0] == 'h')
4139                         instr->type = INSTR_REGWR_RIH;
4140
4141                 instr->regarray.regarray_id = r->id;
4142                 instr->regarray.idx_val = idx_val;
4143                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4144                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4145                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4146                 return 0;
4147         }
4148
4149         /* REGWR_RII. */
4150         src_val = strtoull(src, &src, 0);
4151         CHECK(!src[0], EINVAL);
4152
4153         idx_val = strtoul(idx, &idx, 0);
4154         CHECK(!idx[0], EINVAL);
4155
4156         instr->type = INSTR_REGWR_RII;
4157         instr->regarray.idx_val = idx_val;
4158         instr->regarray.dstsrc_val = src_val;
4159
4160         return 0;
4161 }
4162
4163 static int
4164 instr_regadd_translate(struct rte_swx_pipeline *p,
4165                        struct action *action,
4166                        char **tokens,
4167                        int n_tokens,
4168                        struct instruction *instr,
4169                        struct instruction_data *data __rte_unused)
4170 {
4171         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4172         struct regarray *r;
4173         struct field *fidx, *fsrc;
4174         uint64_t src_val;
4175         uint32_t idx_struct_id, idx_val, src_struct_id;
4176
4177         CHECK(n_tokens == 4, EINVAL);
4178
4179         r = regarray_find(p, regarray);
4180         CHECK(r, EINVAL);
4181
4182         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4183         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4184         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4185         if (fidx && fsrc) {
4186                 CHECK(!fidx->var_size, EINVAL);
4187                 CHECK(!fsrc->var_size, EINVAL);
4188
4189                 instr->type = INSTR_REGADD_RMM;
4190                 if (idx[0] == 'h' && src[0] != 'h')
4191                         instr->type = INSTR_REGADD_RHM;
4192                 if (idx[0] != 'h' && src[0] == 'h')
4193                         instr->type = INSTR_REGADD_RMH;
4194                 if (idx[0] == 'h' && src[0] == 'h')
4195                         instr->type = INSTR_REGADD_RHH;
4196
4197                 instr->regarray.regarray_id = r->id;
4198                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4199                 instr->regarray.idx.n_bits = fidx->n_bits;
4200                 instr->regarray.idx.offset = fidx->offset / 8;
4201                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4202                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4203                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4204                 return 0;
4205         }
4206
4207         /* REGADD_RHI, REGADD_RMI. */
4208         if (fidx && !fsrc) {
4209                 CHECK(!fidx->var_size, EINVAL);
4210
4211                 src_val = strtoull(src, &src, 0);
4212                 CHECK(!src[0], EINVAL);
4213
4214                 instr->type = INSTR_REGADD_RMI;
4215                 if (idx[0] == 'h')
4216                         instr->type = INSTR_REGADD_RHI;
4217
4218                 instr->regarray.regarray_id = r->id;
4219                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4220                 instr->regarray.idx.n_bits = fidx->n_bits;
4221                 instr->regarray.idx.offset = fidx->offset / 8;
4222                 instr->regarray.dstsrc_val = src_val;
4223                 return 0;
4224         }
4225
4226         /* REGADD_RIH, REGADD_RIM. */
4227         if (!fidx && fsrc) {
4228                 idx_val = strtoul(idx, &idx, 0);
4229                 CHECK(!idx[0], EINVAL);
4230
4231                 CHECK(!fsrc->var_size, EINVAL);
4232
4233                 instr->type = INSTR_REGADD_RIM;
4234                 if (src[0] == 'h')
4235                         instr->type = INSTR_REGADD_RIH;
4236
4237                 instr->regarray.regarray_id = r->id;
4238                 instr->regarray.idx_val = idx_val;
4239                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4240                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4241                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4242                 return 0;
4243         }
4244
4245         /* REGADD_RII. */
4246         src_val = strtoull(src, &src, 0);
4247         CHECK(!src[0], EINVAL);
4248
4249         idx_val = strtoul(idx, &idx, 0);
4250         CHECK(!idx[0], EINVAL);
4251
4252         instr->type = INSTR_REGADD_RII;
4253         instr->regarray.idx_val = idx_val;
4254         instr->regarray.dstsrc_val = src_val;
4255         return 0;
4256 }
4257
4258 static inline uint64_t *
4259 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4260 {
4261         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4262         return r->regarray;
4263 }
4264
4265 static inline uint64_t
4266 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4267 {
4268         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4269
4270         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4271         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4272         uint64_t idx64 = *idx64_ptr;
4273         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4274         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4275
4276         return idx;
4277 }
4278
4279 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4280
4281 static inline uint64_t
4282 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4283 {
4284         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4285
4286         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4287         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4288         uint64_t idx64 = *idx64_ptr;
4289         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4290
4291         return idx;
4292 }
4293
4294 #else
4295
4296 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4297
4298 #endif
4299
4300 static inline uint64_t
4301 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4302 {
4303         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4304
4305         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4306
4307         return idx;
4308 }
4309
4310 static inline uint64_t
4311 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4312 {
4313         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4314         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4315         uint64_t src64 = *src64_ptr;
4316         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4317         uint64_t src = src64 & src64_mask;
4318
4319         return src;
4320 }
4321
4322 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4323
4324 static inline uint64_t
4325 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4326 {
4327         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4328         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4329         uint64_t src64 = *src64_ptr;
4330         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4331
4332         return src;
4333 }
4334
4335 #else
4336
4337 #define instr_regarray_src_nbo instr_regarray_src_hbo
4338
4339 #endif
4340
4341 static inline void
4342 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4343 {
4344         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4345         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4346         uint64_t dst64 = *dst64_ptr;
4347         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4348
4349         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4350
4351 }
4352
4353 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4354
4355 static inline void
4356 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4357 {
4358         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4359         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4360         uint64_t dst64 = *dst64_ptr;
4361         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4362
4363         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4364         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4365 }
4366
4367 #else
4368
4369 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4370
4371 #endif
4372
4373 static inline void
4374 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4375 {
4376         struct thread *t = &p->threads[p->thread_id];
4377         struct instruction *ip = t->ip;
4378         uint64_t *regarray, idx;
4379
4380         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4381
4382         /* Structs. */
4383         regarray = instr_regarray_regarray(p, ip);
4384         idx = instr_regarray_idx_nbo(p, t, ip);
4385         rte_prefetch0(&regarray[idx]);
4386
4387         /* Thread. */
4388         thread_ip_inc(p);
4389 }
4390
4391 static inline void
4392 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4393 {
4394         struct thread *t = &p->threads[p->thread_id];
4395         struct instruction *ip = t->ip;
4396         uint64_t *regarray, idx;
4397
4398         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4399
4400         /* Structs. */
4401         regarray = instr_regarray_regarray(p, ip);
4402         idx = instr_regarray_idx_hbo(p, t, ip);
4403         rte_prefetch0(&regarray[idx]);
4404
4405         /* Thread. */
4406         thread_ip_inc(p);
4407 }
4408
4409 static inline void
4410 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4411 {
4412         struct thread *t = &p->threads[p->thread_id];
4413         struct instruction *ip = t->ip;
4414         uint64_t *regarray, idx;
4415
4416         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4417
4418         /* Structs. */
4419         regarray = instr_regarray_regarray(p, ip);
4420         idx = instr_regarray_idx_imm(p, ip);
4421         rte_prefetch0(&regarray[idx]);
4422
4423         /* Thread. */
4424         thread_ip_inc(p);
4425 }
4426
4427 static inline void
4428 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4429 {
4430         struct thread *t = &p->threads[p->thread_id];
4431         struct instruction *ip = t->ip;
4432         uint64_t *regarray, idx;
4433
4434         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4435
4436         /* Structs. */
4437         regarray = instr_regarray_regarray(p, ip);
4438         idx = instr_regarray_idx_nbo(p, t, ip);
4439         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4440
4441         /* Thread. */
4442         thread_ip_inc(p);
4443 }
4444
4445 static inline void
4446 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4447 {
4448         struct thread *t = &p->threads[p->thread_id];
4449         struct instruction *ip = t->ip;
4450         uint64_t *regarray, idx;
4451
4452         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4453
4454         /* Structs. */
4455         regarray = instr_regarray_regarray(p, ip);
4456         idx = instr_regarray_idx_hbo(p, t, ip);
4457         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4458
4459         /* Thread. */
4460         thread_ip_inc(p);
4461 }
4462
4463 static inline void
4464 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4465 {
4466         struct thread *t = &p->threads[p->thread_id];
4467         struct instruction *ip = t->ip;
4468         uint64_t *regarray, idx;
4469
4470         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4471
4472         /* Structs. */
4473         regarray = instr_regarray_regarray(p, ip);
4474         idx = instr_regarray_idx_nbo(p, t, ip);
4475         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4476
4477         /* Thread. */
4478         thread_ip_inc(p);
4479 }
4480
4481 static inline void
4482 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4483 {
4484         struct thread *t = &p->threads[p->thread_id];
4485         struct instruction *ip = t->ip;
4486         uint64_t *regarray, idx;
4487
4488         /* Structs. */
4489         regarray = instr_regarray_regarray(p, ip);
4490         idx = instr_regarray_idx_hbo(p, t, ip);
4491         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4492
4493         /* Thread. */
4494         thread_ip_inc(p);
4495 }
4496
4497 static inline void
4498 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4499 {
4500         struct thread *t = &p->threads[p->thread_id];
4501         struct instruction *ip = t->ip;
4502         uint64_t *regarray, idx;
4503
4504         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4505
4506         /* Structs. */
4507         regarray = instr_regarray_regarray(p, ip);
4508         idx = instr_regarray_idx_imm(p, ip);
4509         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4510
4511         /* Thread. */
4512         thread_ip_inc(p);
4513 }
4514
4515 static inline void
4516 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4517 {
4518         struct thread *t = &p->threads[p->thread_id];
4519         struct instruction *ip = t->ip;
4520         uint64_t *regarray, idx;
4521
4522         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4523
4524         /* Structs. */
4525         regarray = instr_regarray_regarray(p, ip);
4526         idx = instr_regarray_idx_imm(p, ip);
4527         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4528
4529         /* Thread. */
4530         thread_ip_inc(p);
4531 }
4532
4533 static inline void
4534 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4535 {
4536         struct thread *t = &p->threads[p->thread_id];
4537         struct instruction *ip = t->ip;
4538         uint64_t *regarray, idx, src;
4539
4540         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4541
4542         /* Structs. */
4543         regarray = instr_regarray_regarray(p, ip);
4544         idx = instr_regarray_idx_nbo(p, t, ip);
4545         src = instr_regarray_src_nbo(t, ip);
4546         regarray[idx] = src;
4547
4548         /* Thread. */
4549         thread_ip_inc(p);
4550 }
4551
4552 static inline void
4553 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4554 {
4555         struct thread *t = &p->threads[p->thread_id];
4556         struct instruction *ip = t->ip;
4557         uint64_t *regarray, idx, src;
4558
4559         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4560
4561         /* Structs. */
4562         regarray = instr_regarray_regarray(p, ip);
4563         idx = instr_regarray_idx_nbo(p, t, ip);
4564         src = instr_regarray_src_hbo(t, ip);
4565         regarray[idx] = src;
4566
4567         /* Thread. */
4568         thread_ip_inc(p);
4569 }
4570
4571 static inline void
4572 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4573 {
4574         struct thread *t = &p->threads[p->thread_id];
4575         struct instruction *ip = t->ip;
4576         uint64_t *regarray, idx, src;
4577
4578         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4579
4580         /* Structs. */
4581         regarray = instr_regarray_regarray(p, ip);
4582         idx = instr_regarray_idx_hbo(p, t, ip);
4583         src = instr_regarray_src_nbo(t, ip);
4584         regarray[idx] = src;
4585
4586         /* Thread. */
4587         thread_ip_inc(p);
4588 }
4589
4590 static inline void
4591 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4592 {
4593         struct thread *t = &p->threads[p->thread_id];
4594         struct instruction *ip = t->ip;
4595         uint64_t *regarray, idx, src;
4596
4597         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4598
4599         /* Structs. */
4600         regarray = instr_regarray_regarray(p, ip);
4601         idx = instr_regarray_idx_hbo(p, t, ip);
4602         src = instr_regarray_src_hbo(t, ip);
4603         regarray[idx] = src;
4604
4605         /* Thread. */
4606         thread_ip_inc(p);
4607 }
4608
4609 static inline void
4610 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4611 {
4612         struct thread *t = &p->threads[p->thread_id];
4613         struct instruction *ip = t->ip;
4614         uint64_t *regarray, idx, src;
4615
4616         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4617
4618         /* Structs. */
4619         regarray = instr_regarray_regarray(p, ip);
4620         idx = instr_regarray_idx_nbo(p, t, ip);
4621         src = ip->regarray.dstsrc_val;
4622         regarray[idx] = src;
4623
4624         /* Thread. */
4625         thread_ip_inc(p);
4626 }
4627
4628 static inline void
4629 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4630 {
4631         struct thread *t = &p->threads[p->thread_id];
4632         struct instruction *ip = t->ip;
4633         uint64_t *regarray, idx, src;
4634
4635         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4636
4637         /* Structs. */
4638         regarray = instr_regarray_regarray(p, ip);
4639         idx = instr_regarray_idx_hbo(p, t, ip);
4640         src = ip->regarray.dstsrc_val;
4641         regarray[idx] = src;
4642
4643         /* Thread. */
4644         thread_ip_inc(p);
4645 }
4646
4647 static inline void
4648 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4649 {
4650         struct thread *t = &p->threads[p->thread_id];
4651         struct instruction *ip = t->ip;
4652         uint64_t *regarray, idx, src;
4653
4654         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4655
4656         /* Structs. */
4657         regarray = instr_regarray_regarray(p, ip);
4658         idx = instr_regarray_idx_imm(p, ip);
4659         src = instr_regarray_src_nbo(t, ip);
4660         regarray[idx] = src;
4661
4662         /* Thread. */
4663         thread_ip_inc(p);
4664 }
4665
4666 static inline void
4667 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4668 {
4669         struct thread *t = &p->threads[p->thread_id];
4670         struct instruction *ip = t->ip;
4671         uint64_t *regarray, idx, src;
4672
4673         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4674
4675         /* Structs. */
4676         regarray = instr_regarray_regarray(p, ip);
4677         idx = instr_regarray_idx_imm(p, ip);
4678         src = instr_regarray_src_hbo(t, ip);
4679         regarray[idx] = src;
4680
4681         /* Thread. */
4682         thread_ip_inc(p);
4683 }
4684
4685 static inline void
4686 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4687 {
4688         struct thread *t = &p->threads[p->thread_id];
4689         struct instruction *ip = t->ip;
4690         uint64_t *regarray, idx, src;
4691
4692         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4693
4694         /* Structs. */
4695         regarray = instr_regarray_regarray(p, ip);
4696         idx = instr_regarray_idx_imm(p, ip);
4697         src = ip->regarray.dstsrc_val;
4698         regarray[idx] = src;
4699
4700         /* Thread. */
4701         thread_ip_inc(p);
4702 }
4703
4704 static inline void
4705 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4706 {
4707         struct thread *t = &p->threads[p->thread_id];
4708         struct instruction *ip = t->ip;
4709         uint64_t *regarray, idx, src;
4710
4711         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4712
4713         /* Structs. */
4714         regarray = instr_regarray_regarray(p, ip);
4715         idx = instr_regarray_idx_nbo(p, t, ip);
4716         src = instr_regarray_src_nbo(t, ip);
4717         regarray[idx] += src;
4718
4719         /* Thread. */
4720         thread_ip_inc(p);
4721 }
4722
4723 static inline void
4724 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4725 {
4726         struct thread *t = &p->threads[p->thread_id];
4727         struct instruction *ip = t->ip;
4728         uint64_t *regarray, idx, src;
4729
4730         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4731
4732         /* Structs. */
4733         regarray = instr_regarray_regarray(p, ip);
4734         idx = instr_regarray_idx_nbo(p, t, ip);
4735         src = instr_regarray_src_hbo(t, ip);
4736         regarray[idx] += src;
4737
4738         /* Thread. */
4739         thread_ip_inc(p);
4740 }
4741
4742 static inline void
4743 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4744 {
4745         struct thread *t = &p->threads[p->thread_id];
4746         struct instruction *ip = t->ip;
4747         uint64_t *regarray, idx, src;
4748
4749         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4750
4751         /* Structs. */
4752         regarray = instr_regarray_regarray(p, ip);
4753         idx = instr_regarray_idx_hbo(p, t, ip);
4754         src = instr_regarray_src_nbo(t, ip);
4755         regarray[idx] += src;
4756
4757         /* Thread. */
4758         thread_ip_inc(p);
4759 }
4760
4761 static inline void
4762 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4763 {
4764         struct thread *t = &p->threads[p->thread_id];
4765         struct instruction *ip = t->ip;
4766         uint64_t *regarray, idx, src;
4767
4768         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4769
4770         /* Structs. */
4771         regarray = instr_regarray_regarray(p, ip);
4772         idx = instr_regarray_idx_hbo(p, t, ip);
4773         src = instr_regarray_src_hbo(t, ip);
4774         regarray[idx] += src;
4775
4776         /* Thread. */
4777         thread_ip_inc(p);
4778 }
4779
4780 static inline void
4781 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4782 {
4783         struct thread *t = &p->threads[p->thread_id];
4784         struct instruction *ip = t->ip;
4785         uint64_t *regarray, idx, src;
4786
4787         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4788
4789         /* Structs. */
4790         regarray = instr_regarray_regarray(p, ip);
4791         idx = instr_regarray_idx_nbo(p, t, ip);
4792         src = ip->regarray.dstsrc_val;
4793         regarray[idx] += src;
4794
4795         /* Thread. */
4796         thread_ip_inc(p);
4797 }
4798
4799 static inline void
4800 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4801 {
4802         struct thread *t = &p->threads[p->thread_id];
4803         struct instruction *ip = t->ip;
4804         uint64_t *regarray, idx, src;
4805
4806         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4807
4808         /* Structs. */
4809         regarray = instr_regarray_regarray(p, ip);
4810         idx = instr_regarray_idx_hbo(p, t, ip);
4811         src = ip->regarray.dstsrc_val;
4812         regarray[idx] += src;
4813
4814         /* Thread. */
4815         thread_ip_inc(p);
4816 }
4817
4818 static inline void
4819 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4820 {
4821         struct thread *t = &p->threads[p->thread_id];
4822         struct instruction *ip = t->ip;
4823         uint64_t *regarray, idx, src;
4824
4825         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4826
4827         /* Structs. */
4828         regarray = instr_regarray_regarray(p, ip);
4829         idx = instr_regarray_idx_imm(p, ip);
4830         src = instr_regarray_src_nbo(t, ip);
4831         regarray[idx] += src;
4832
4833         /* Thread. */
4834         thread_ip_inc(p);
4835 }
4836
4837 static inline void
4838 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4839 {
4840         struct thread *t = &p->threads[p->thread_id];
4841         struct instruction *ip = t->ip;
4842         uint64_t *regarray, idx, src;
4843
4844         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4845
4846         /* Structs. */
4847         regarray = instr_regarray_regarray(p, ip);
4848         idx = instr_regarray_idx_imm(p, ip);
4849         src = instr_regarray_src_hbo(t, ip);
4850         regarray[idx] += src;
4851
4852         /* Thread. */
4853         thread_ip_inc(p);
4854 }
4855
4856 static inline void
4857 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4858 {
4859         struct thread *t = &p->threads[p->thread_id];
4860         struct instruction *ip = t->ip;
4861         uint64_t *regarray, idx, src;
4862
4863         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4864
4865         /* Structs. */
4866         regarray = instr_regarray_regarray(p, ip);
4867         idx = instr_regarray_idx_imm(p, ip);
4868         src = ip->regarray.dstsrc_val;
4869         regarray[idx] += src;
4870
4871         /* Thread. */
4872         thread_ip_inc(p);
4873 }
4874
4875 /*
4876  * metarray.
4877  */
4878 static struct metarray *
4879 metarray_find(struct rte_swx_pipeline *p, const char *name);
4880
4881 static int
4882 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4883                             struct action *action,
4884                             char **tokens,
4885                             int n_tokens,
4886                             struct instruction *instr,
4887                             struct instruction_data *data __rte_unused)
4888 {
4889         char *metarray = tokens[1], *idx = tokens[2];
4890         struct metarray *m;
4891         struct field *fidx;
4892         uint32_t idx_struct_id, idx_val;
4893
4894         CHECK(n_tokens == 3, EINVAL);
4895
4896         m = metarray_find(p, metarray);
4897         CHECK(m, EINVAL);
4898
4899         /* METPREFETCH_H, METPREFETCH_M. */
4900         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4901         if (fidx) {
4902                 CHECK(!fidx->var_size, EINVAL);
4903
4904                 instr->type = INSTR_METPREFETCH_M;
4905                 if (idx[0] == 'h')
4906                         instr->type = INSTR_METPREFETCH_H;
4907
4908                 instr->meter.metarray_id = m->id;
4909                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4910                 instr->meter.idx.n_bits = fidx->n_bits;
4911                 instr->meter.idx.offset = fidx->offset / 8;
4912                 return 0;
4913         }
4914
4915         /* METPREFETCH_I. */
4916         idx_val = strtoul(idx, &idx, 0);
4917         CHECK(!idx[0], EINVAL);
4918
4919         instr->type = INSTR_METPREFETCH_I;
4920         instr->meter.metarray_id = m->id;
4921         instr->meter.idx_val = idx_val;
4922         return 0;
4923 }
4924
4925 static int
4926 instr_meter_translate(struct rte_swx_pipeline *p,
4927                       struct action *action,
4928                       char **tokens,
4929                       int n_tokens,
4930                       struct instruction *instr,
4931                       struct instruction_data *data __rte_unused)
4932 {
4933         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4934         char *color_in = tokens[4], *color_out = tokens[5];
4935         struct metarray *m;
4936         struct field *fidx, *flength, *fcin, *fcout;
4937         uint32_t idx_struct_id, length_struct_id;
4938         uint32_t color_in_struct_id, color_out_struct_id;
4939
4940         CHECK(n_tokens == 6, EINVAL);
4941
4942         m = metarray_find(p, metarray);
4943         CHECK(m, EINVAL);
4944
4945         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4946
4947         flength = struct_field_parse(p, action, length, &length_struct_id);
4948         CHECK(flength, EINVAL);
4949         CHECK(!flength->var_size, EINVAL);
4950
4951         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4952
4953         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4954         CHECK(fcout, EINVAL);
4955         CHECK(!fcout->var_size, EINVAL);
4956
4957         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4958         if (fidx && fcin) {
4959                 CHECK(!fidx->var_size, EINVAL);
4960                 CHECK(!fcin->var_size, EINVAL);
4961
4962                 instr->type = INSTR_METER_MMM;
4963                 if (idx[0] == 'h' && length[0] == 'h')
4964                         instr->type = INSTR_METER_HHM;
4965                 if (idx[0] == 'h' && length[0] != 'h')
4966                         instr->type = INSTR_METER_HMM;
4967                 if (idx[0] != 'h' && length[0] == 'h')
4968                         instr->type = INSTR_METER_MHM;
4969
4970                 instr->meter.metarray_id = m->id;
4971
4972                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4973                 instr->meter.idx.n_bits = fidx->n_bits;
4974                 instr->meter.idx.offset = fidx->offset / 8;
4975
4976                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4977                 instr->meter.length.n_bits = flength->n_bits;
4978                 instr->meter.length.offset = flength->offset / 8;
4979
4980                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4981                 instr->meter.color_in.n_bits = fcin->n_bits;
4982                 instr->meter.color_in.offset = fcin->offset / 8;
4983
4984                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4985                 instr->meter.color_out.n_bits = fcout->n_bits;
4986                 instr->meter.color_out.offset = fcout->offset / 8;
4987
4988                 return 0;
4989         }
4990
4991         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4992         if (fidx && !fcin) {
4993                 uint32_t color_in_val;
4994
4995                 CHECK(!fidx->var_size, EINVAL);
4996
4997                 color_in_val = strtoul(color_in, &color_in, 0);
4998                 CHECK(!color_in[0], EINVAL);
4999
5000                 instr->type = INSTR_METER_MMI;
5001                 if (idx[0] == 'h' && length[0] == 'h')
5002                         instr->type = INSTR_METER_HHI;
5003                 if (idx[0] == 'h' && length[0] != 'h')
5004                         instr->type = INSTR_METER_HMI;
5005                 if (idx[0] != 'h' && length[0] == 'h')
5006                         instr->type = INSTR_METER_MHI;
5007
5008                 instr->meter.metarray_id = m->id;
5009
5010                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5011                 instr->meter.idx.n_bits = fidx->n_bits;
5012                 instr->meter.idx.offset = fidx->offset / 8;
5013
5014                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5015                 instr->meter.length.n_bits = flength->n_bits;
5016                 instr->meter.length.offset = flength->offset / 8;
5017
5018                 instr->meter.color_in_val = color_in_val;
5019
5020                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5021                 instr->meter.color_out.n_bits = fcout->n_bits;
5022                 instr->meter.color_out.offset = fcout->offset / 8;
5023
5024                 return 0;
5025         }
5026
5027         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5028         if (!fidx && fcin) {
5029                 uint32_t idx_val;
5030
5031                 idx_val = strtoul(idx, &idx, 0);
5032                 CHECK(!idx[0], EINVAL);
5033
5034                 CHECK(!fcin->var_size, EINVAL);
5035
5036                 instr->type = INSTR_METER_IMM;
5037                 if (length[0] == 'h')
5038                         instr->type = INSTR_METER_IHM;
5039
5040                 instr->meter.metarray_id = m->id;
5041
5042                 instr->meter.idx_val = idx_val;
5043
5044                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5045                 instr->meter.length.n_bits = flength->n_bits;
5046                 instr->meter.length.offset = flength->offset / 8;
5047
5048                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5049                 instr->meter.color_in.n_bits = fcin->n_bits;
5050                 instr->meter.color_in.offset = fcin->offset / 8;
5051
5052                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5053                 instr->meter.color_out.n_bits = fcout->n_bits;
5054                 instr->meter.color_out.offset = fcout->offset / 8;
5055
5056                 return 0;
5057         }
5058
5059         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5060         if (!fidx && !fcin) {
5061                 uint32_t idx_val, color_in_val;
5062
5063                 idx_val = strtoul(idx, &idx, 0);
5064                 CHECK(!idx[0], EINVAL);
5065
5066                 color_in_val = strtoul(color_in, &color_in, 0);
5067                 CHECK(!color_in[0], EINVAL);
5068
5069                 instr->type = INSTR_METER_IMI;
5070                 if (length[0] == 'h')
5071                         instr->type = INSTR_METER_IHI;
5072
5073                 instr->meter.metarray_id = m->id;
5074
5075                 instr->meter.idx_val = idx_val;
5076
5077                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5078                 instr->meter.length.n_bits = flength->n_bits;
5079                 instr->meter.length.offset = flength->offset / 8;
5080
5081                 instr->meter.color_in_val = color_in_val;
5082
5083                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5084                 instr->meter.color_out.n_bits = fcout->n_bits;
5085                 instr->meter.color_out.offset = fcout->offset / 8;
5086
5087                 return 0;
5088         }
5089
5090         CHECK(0, EINVAL);
5091 }
5092
5093 static inline struct meter *
5094 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5095 {
5096         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5097
5098         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5099         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5100         uint64_t idx64 = *idx64_ptr;
5101         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5102         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5103
5104         return &r->metarray[idx];
5105 }
5106
5107 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5108
5109 static inline struct meter *
5110 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5111 {
5112         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5113
5114         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5115         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5116         uint64_t idx64 = *idx64_ptr;
5117         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5118
5119         return &r->metarray[idx];
5120 }
5121
5122 #else
5123
5124 #define instr_meter_idx_nbo instr_meter_idx_hbo
5125
5126 #endif
5127
5128 static inline struct meter *
5129 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5130 {
5131         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5132
5133         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5134
5135         return &r->metarray[idx];
5136 }
5137
5138 static inline uint32_t
5139 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5140 {
5141         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5142         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5143         uint64_t src64 = *src64_ptr;
5144         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5145         uint64_t src = src64 & src64_mask;
5146
5147         return (uint32_t)src;
5148 }
5149
5150 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5151
5152 static inline uint32_t
5153 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5154 {
5155         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5156         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5157         uint64_t src64 = *src64_ptr;
5158         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5159
5160         return (uint32_t)src;
5161 }
5162
5163 #else
5164
5165 #define instr_meter_length_nbo instr_meter_length_hbo
5166
5167 #endif
5168
5169 static inline enum rte_color
5170 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5171 {
5172         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5173         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5174         uint64_t src64 = *src64_ptr;
5175         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5176         uint64_t src = src64 & src64_mask;
5177
5178         return (enum rte_color)src;
5179 }
5180
5181 static inline void
5182 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5183 {
5184         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5185         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5186         uint64_t dst64 = *dst64_ptr;
5187         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5188
5189         uint64_t src = (uint64_t)color_out;
5190
5191         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5192 }
5193
5194 static inline void
5195 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5196 {
5197         struct thread *t = &p->threads[p->thread_id];
5198         struct instruction *ip = t->ip;
5199         struct meter *m;
5200
5201         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5202
5203         /* Structs. */
5204         m = instr_meter_idx_nbo(p, t, ip);
5205         rte_prefetch0(m);
5206
5207         /* Thread. */
5208         thread_ip_inc(p);
5209 }
5210
5211 static inline void
5212 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5213 {
5214         struct thread *t = &p->threads[p->thread_id];
5215         struct instruction *ip = t->ip;
5216         struct meter *m;
5217
5218         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5219
5220         /* Structs. */
5221         m = instr_meter_idx_hbo(p, t, ip);
5222         rte_prefetch0(m);
5223
5224         /* Thread. */
5225         thread_ip_inc(p);
5226 }
5227
5228 static inline void
5229 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5230 {
5231         struct thread *t = &p->threads[p->thread_id];
5232         struct instruction *ip = t->ip;
5233         struct meter *m;
5234
5235         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5236
5237         /* Structs. */
5238         m = instr_meter_idx_imm(p, ip);
5239         rte_prefetch0(m);
5240
5241         /* Thread. */
5242         thread_ip_inc(p);
5243 }
5244
5245 static inline void
5246 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5247 {
5248         struct thread *t = &p->threads[p->thread_id];
5249         struct instruction *ip = t->ip;
5250         struct meter *m;
5251         uint64_t time, n_pkts, n_bytes;
5252         uint32_t length;
5253         enum rte_color color_in, color_out;
5254
5255         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5256
5257         /* Structs. */
5258         m = instr_meter_idx_nbo(p, t, ip);
5259         rte_prefetch0(m->n_pkts);
5260         time = rte_get_tsc_cycles();
5261         length = instr_meter_length_nbo(t, ip);
5262         color_in = instr_meter_color_in_hbo(t, ip);
5263
5264         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5265                 &m->profile->profile,
5266                 time,
5267                 length,
5268                 color_in);
5269
5270         color_out &= m->color_mask;
5271
5272         n_pkts = m->n_pkts[color_out];
5273         n_bytes = m->n_bytes[color_out];
5274
5275         instr_meter_color_out_hbo_set(t, ip, color_out);
5276
5277         m->n_pkts[color_out] = n_pkts + 1;
5278         m->n_bytes[color_out] = n_bytes + length;
5279
5280         /* Thread. */
5281         thread_ip_inc(p);
5282 }
5283
5284 static inline void
5285 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5286 {
5287         struct thread *t = &p->threads[p->thread_id];
5288         struct instruction *ip = t->ip;
5289         struct meter *m;
5290         uint64_t time, n_pkts, n_bytes;
5291         uint32_t length;
5292         enum rte_color color_in, color_out;
5293
5294         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5295
5296         /* Structs. */
5297         m = instr_meter_idx_nbo(p, t, ip);
5298         rte_prefetch0(m->n_pkts);
5299         time = rte_get_tsc_cycles();
5300         length = instr_meter_length_nbo(t, ip);
5301         color_in = (enum rte_color)ip->meter.color_in_val;
5302
5303         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5304                 &m->profile->profile,
5305                 time,
5306                 length,
5307                 color_in);
5308
5309         color_out &= m->color_mask;
5310
5311         n_pkts = m->n_pkts[color_out];
5312         n_bytes = m->n_bytes[color_out];
5313
5314         instr_meter_color_out_hbo_set(t, ip, color_out);
5315
5316         m->n_pkts[color_out] = n_pkts + 1;
5317         m->n_bytes[color_out] = n_bytes + length;
5318
5319         /* Thread. */
5320         thread_ip_inc(p);
5321 }
5322
5323 static inline void
5324 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5325 {
5326         struct thread *t = &p->threads[p->thread_id];
5327         struct instruction *ip = t->ip;
5328         struct meter *m;
5329         uint64_t time, n_pkts, n_bytes;
5330         uint32_t length;
5331         enum rte_color color_in, color_out;
5332
5333         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5334
5335         /* Structs. */
5336         m = instr_meter_idx_nbo(p, t, ip);
5337         rte_prefetch0(m->n_pkts);
5338         time = rte_get_tsc_cycles();
5339         length = instr_meter_length_hbo(t, ip);
5340         color_in = instr_meter_color_in_hbo(t, ip);
5341
5342         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5343                 &m->profile->profile,
5344                 time,
5345                 length,
5346                 color_in);
5347
5348         color_out &= m->color_mask;
5349
5350         n_pkts = m->n_pkts[color_out];
5351         n_bytes = m->n_bytes[color_out];
5352
5353         instr_meter_color_out_hbo_set(t, ip, color_out);
5354
5355         m->n_pkts[color_out] = n_pkts + 1;
5356         m->n_bytes[color_out] = n_bytes + length;
5357
5358         /* Thread. */
5359         thread_ip_inc(p);
5360 }
5361 static inline void
5362 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5363 {
5364         struct thread *t = &p->threads[p->thread_id];
5365         struct instruction *ip = t->ip;
5366         struct meter *m;
5367         uint64_t time, n_pkts, n_bytes;
5368         uint32_t length;
5369         enum rte_color color_in, color_out;
5370
5371         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5372
5373         /* Structs. */
5374         m = instr_meter_idx_nbo(p, t, ip);
5375         rte_prefetch0(m->n_pkts);
5376         time = rte_get_tsc_cycles();
5377         length = instr_meter_length_hbo(t, ip);
5378         color_in = (enum rte_color)ip->meter.color_in_val;
5379
5380         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5381                 &m->profile->profile,
5382                 time,
5383                 length,
5384                 color_in);
5385
5386         color_out &= m->color_mask;
5387
5388         n_pkts = m->n_pkts[color_out];
5389         n_bytes = m->n_bytes[color_out];
5390
5391         instr_meter_color_out_hbo_set(t, ip, color_out);
5392
5393         m->n_pkts[color_out] = n_pkts + 1;
5394         m->n_bytes[color_out] = n_bytes + length;
5395
5396         /* Thread. */
5397         thread_ip_inc(p);
5398 }
5399
5400 static inline void
5401 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5402 {
5403         struct thread *t = &p->threads[p->thread_id];
5404         struct instruction *ip = t->ip;
5405         struct meter *m;
5406         uint64_t time, n_pkts, n_bytes;
5407         uint32_t length;
5408         enum rte_color color_in, color_out;
5409
5410         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5411
5412         /* Structs. */
5413         m = instr_meter_idx_hbo(p, t, ip);
5414         rte_prefetch0(m->n_pkts);
5415         time = rte_get_tsc_cycles();
5416         length = instr_meter_length_nbo(t, ip);
5417         color_in = instr_meter_color_in_hbo(t, ip);
5418
5419         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5420                 &m->profile->profile,
5421                 time,
5422                 length,
5423                 color_in);
5424
5425         color_out &= m->color_mask;
5426
5427         n_pkts = m->n_pkts[color_out];
5428         n_bytes = m->n_bytes[color_out];
5429
5430         instr_meter_color_out_hbo_set(t, ip, color_out);
5431
5432         m->n_pkts[color_out] = n_pkts + 1;
5433         m->n_bytes[color_out] = n_bytes + length;
5434
5435         /* Thread. */
5436         thread_ip_inc(p);
5437 }
5438
5439 static inline void
5440 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5441 {
5442         struct thread *t = &p->threads[p->thread_id];
5443         struct instruction *ip = t->ip;
5444         struct meter *m;
5445         uint64_t time, n_pkts, n_bytes;
5446         uint32_t length;
5447         enum rte_color color_in, color_out;
5448
5449         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5450
5451         /* Structs. */
5452         m = instr_meter_idx_hbo(p, t, ip);
5453         rte_prefetch0(m->n_pkts);
5454         time = rte_get_tsc_cycles();
5455         length = instr_meter_length_nbo(t, ip);
5456         color_in = (enum rte_color)ip->meter.color_in_val;
5457
5458         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5459                 &m->profile->profile,
5460                 time,
5461                 length,
5462                 color_in);
5463
5464         color_out &= m->color_mask;
5465
5466         n_pkts = m->n_pkts[color_out];
5467         n_bytes = m->n_bytes[color_out];
5468
5469         instr_meter_color_out_hbo_set(t, ip, color_out);
5470
5471         m->n_pkts[color_out] = n_pkts + 1;
5472         m->n_bytes[color_out] = n_bytes + length;
5473
5474         /* Thread. */
5475         thread_ip_inc(p);
5476 }
5477
5478 static inline void
5479 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5480 {
5481         struct thread *t = &p->threads[p->thread_id];
5482         struct instruction *ip = t->ip;
5483         struct meter *m;
5484         uint64_t time, n_pkts, n_bytes;
5485         uint32_t length;
5486         enum rte_color color_in, color_out;
5487
5488         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5489
5490         /* Structs. */
5491         m = instr_meter_idx_hbo(p, t, ip);
5492         rte_prefetch0(m->n_pkts);
5493         time = rte_get_tsc_cycles();
5494         length = instr_meter_length_hbo(t, ip);
5495         color_in = instr_meter_color_in_hbo(t, ip);
5496
5497         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5498                 &m->profile->profile,
5499                 time,
5500                 length,
5501                 color_in);
5502
5503         color_out &= m->color_mask;
5504
5505         n_pkts = m->n_pkts[color_out];
5506         n_bytes = m->n_bytes[color_out];
5507
5508         instr_meter_color_out_hbo_set(t, ip, color_out);
5509
5510         m->n_pkts[color_out] = n_pkts + 1;
5511         m->n_bytes[color_out] = n_bytes + length;
5512
5513         /* Thread. */
5514         thread_ip_inc(p);
5515 }
5516
5517 static inline void
5518 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5519 {
5520         struct thread *t = &p->threads[p->thread_id];
5521         struct instruction *ip = t->ip;
5522         struct meter *m;
5523         uint64_t time, n_pkts, n_bytes;
5524         uint32_t length;
5525         enum rte_color color_in, color_out;
5526
5527         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5528
5529         /* Structs. */
5530         m = instr_meter_idx_hbo(p, t, ip);
5531         rte_prefetch0(m->n_pkts);
5532         time = rte_get_tsc_cycles();
5533         length = instr_meter_length_hbo(t, ip);
5534         color_in = (enum rte_color)ip->meter.color_in_val;
5535
5536         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5537                 &m->profile->profile,
5538                 time,
5539                 length,
5540                 color_in);
5541
5542         color_out &= m->color_mask;
5543
5544         n_pkts = m->n_pkts[color_out];
5545         n_bytes = m->n_bytes[color_out];
5546
5547         instr_meter_color_out_hbo_set(t, ip, color_out);
5548
5549         m->n_pkts[color_out] = n_pkts + 1;
5550         m->n_bytes[color_out] = n_bytes + length;
5551
5552         /* Thread. */
5553         thread_ip_inc(p);
5554 }
5555
5556 static inline void
5557 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5558 {
5559         struct thread *t = &p->threads[p->thread_id];
5560         struct instruction *ip = t->ip;
5561         struct meter *m;
5562         uint64_t time, n_pkts, n_bytes;
5563         uint32_t length;
5564         enum rte_color color_in, color_out;
5565
5566         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5567
5568         /* Structs. */
5569         m = instr_meter_idx_imm(p, ip);
5570         rte_prefetch0(m->n_pkts);
5571         time = rte_get_tsc_cycles();
5572         length = instr_meter_length_nbo(t, ip);
5573         color_in = instr_meter_color_in_hbo(t, ip);
5574
5575         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5576                 &m->profile->profile,
5577                 time,
5578                 length,
5579                 color_in);
5580
5581         color_out &= m->color_mask;
5582
5583         n_pkts = m->n_pkts[color_out];
5584         n_bytes = m->n_bytes[color_out];
5585
5586         instr_meter_color_out_hbo_set(t, ip, color_out);
5587
5588         m->n_pkts[color_out] = n_pkts + 1;
5589         m->n_bytes[color_out] = n_bytes + length;
5590
5591         /* Thread. */
5592         thread_ip_inc(p);
5593 }
5594
5595 static inline void
5596 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5597 {
5598         struct thread *t = &p->threads[p->thread_id];
5599         struct instruction *ip = t->ip;
5600         struct meter *m;
5601         uint64_t time, n_pkts, n_bytes;
5602         uint32_t length;
5603         enum rte_color color_in, color_out;
5604
5605         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5606
5607         /* Structs. */
5608         m = instr_meter_idx_imm(p, ip);
5609         rte_prefetch0(m->n_pkts);
5610         time = rte_get_tsc_cycles();
5611         length = instr_meter_length_nbo(t, ip);
5612         color_in = (enum rte_color)ip->meter.color_in_val;
5613
5614         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5615                 &m->profile->profile,
5616                 time,
5617                 length,
5618                 color_in);
5619
5620         color_out &= m->color_mask;
5621
5622         n_pkts = m->n_pkts[color_out];
5623         n_bytes = m->n_bytes[color_out];
5624
5625         instr_meter_color_out_hbo_set(t, ip, color_out);
5626
5627         m->n_pkts[color_out] = n_pkts + 1;
5628         m->n_bytes[color_out] = n_bytes + length;
5629
5630         /* Thread. */
5631         thread_ip_inc(p);
5632 }
5633
5634 static inline void
5635 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5636 {
5637         struct thread *t = &p->threads[p->thread_id];
5638         struct instruction *ip = t->ip;
5639         struct meter *m;
5640         uint64_t time, n_pkts, n_bytes;
5641         uint32_t length;
5642         enum rte_color color_in, color_out;
5643
5644         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5645
5646         /* Structs. */
5647         m = instr_meter_idx_imm(p, ip);
5648         rte_prefetch0(m->n_pkts);
5649         time = rte_get_tsc_cycles();
5650         length = instr_meter_length_hbo(t, ip);
5651         color_in = instr_meter_color_in_hbo(t, ip);
5652
5653         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5654                 &m->profile->profile,
5655                 time,
5656                 length,
5657                 color_in);
5658
5659         color_out &= m->color_mask;
5660
5661         n_pkts = m->n_pkts[color_out];
5662         n_bytes = m->n_bytes[color_out];
5663
5664         instr_meter_color_out_hbo_set(t, ip, color_out);
5665
5666         m->n_pkts[color_out] = n_pkts + 1;
5667         m->n_bytes[color_out] = n_bytes + length;
5668
5669         /* Thread. */
5670         thread_ip_inc(p);
5671 }
5672 static inline void
5673 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5674 {
5675         struct thread *t = &p->threads[p->thread_id];
5676         struct instruction *ip = t->ip;
5677         struct meter *m;
5678         uint64_t time, n_pkts, n_bytes;
5679         uint32_t length;
5680         enum rte_color color_in, color_out;
5681
5682         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5683
5684         /* Structs. */
5685         m = instr_meter_idx_imm(p, ip);
5686         rte_prefetch0(m->n_pkts);
5687         time = rte_get_tsc_cycles();
5688         length = instr_meter_length_hbo(t, ip);
5689         color_in = (enum rte_color)ip->meter.color_in_val;
5690
5691         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5692                 &m->profile->profile,
5693                 time,
5694                 length,
5695                 color_in);
5696
5697         color_out &= m->color_mask;
5698
5699         n_pkts = m->n_pkts[color_out];
5700         n_bytes = m->n_bytes[color_out];
5701
5702         instr_meter_color_out_hbo_set(t, ip, color_out);
5703
5704         m->n_pkts[color_out] = n_pkts + 1;
5705         m->n_bytes[color_out] = n_bytes + length;
5706
5707         /* Thread. */
5708         thread_ip_inc(p);
5709 }
5710
5711 /*
5712  * jmp.
5713  */
5714 static int
5715 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5716                     struct action *action __rte_unused,
5717                     char **tokens,
5718                     int n_tokens,
5719                     struct instruction *instr,
5720                     struct instruction_data *data)
5721 {
5722         CHECK(n_tokens == 2, EINVAL);
5723
5724         strcpy(data->jmp_label, tokens[1]);
5725
5726         instr->type = INSTR_JMP;
5727         instr->jmp.ip = NULL; /* Resolved later. */
5728         return 0;
5729 }
5730
5731 static int
5732 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5733                           struct action *action __rte_unused,
5734                           char **tokens,
5735                           int n_tokens,
5736                           struct instruction *instr,
5737                           struct instruction_data *data)
5738 {
5739         struct header *h;
5740
5741         CHECK(n_tokens == 3, EINVAL);
5742
5743         strcpy(data->jmp_label, tokens[1]);
5744
5745         h = header_parse(p, tokens[2]);
5746         CHECK(h, EINVAL);
5747
5748         instr->type = INSTR_JMP_VALID;
5749         instr->jmp.ip = NULL; /* Resolved later. */
5750         instr->jmp.header_id = h->id;
5751         return 0;
5752 }
5753
5754 static int
5755 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5756                             struct action *action __rte_unused,
5757                             char **tokens,
5758                             int n_tokens,
5759                             struct instruction *instr,
5760                             struct instruction_data *data)
5761 {
5762         struct header *h;
5763
5764         CHECK(n_tokens == 3, EINVAL);
5765
5766         strcpy(data->jmp_label, tokens[1]);
5767
5768         h = header_parse(p, tokens[2]);
5769         CHECK(h, EINVAL);
5770
5771         instr->type = INSTR_JMP_INVALID;
5772         instr->jmp.ip = NULL; /* Resolved later. */
5773         instr->jmp.header_id = h->id;
5774         return 0;
5775 }
5776
5777 static int
5778 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5779                         struct action *action,
5780                         char **tokens,
5781                         int n_tokens,
5782                         struct instruction *instr,
5783                         struct instruction_data *data)
5784 {
5785         CHECK(!action, EINVAL);
5786         CHECK(n_tokens == 2, EINVAL);
5787
5788         strcpy(data->jmp_label, tokens[1]);
5789
5790         instr->type = INSTR_JMP_HIT;
5791         instr->jmp.ip = NULL; /* Resolved later. */
5792         return 0;
5793 }
5794
5795 static int
5796 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5797                          struct action *action,
5798                          char **tokens,
5799                          int n_tokens,
5800                          struct instruction *instr,
5801                          struct instruction_data *data)
5802 {
5803         CHECK(!action, EINVAL);
5804         CHECK(n_tokens == 2, EINVAL);
5805
5806         strcpy(data->jmp_label, tokens[1]);
5807
5808         instr->type = INSTR_JMP_MISS;
5809         instr->jmp.ip = NULL; /* Resolved later. */
5810         return 0;
5811 }
5812
5813 static int
5814 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5815                                struct action *action,
5816                                char **tokens,
5817                                int n_tokens,
5818                                struct instruction *instr,
5819                                struct instruction_data *data)
5820 {
5821         struct action *a;
5822
5823         CHECK(!action, EINVAL);
5824         CHECK(n_tokens == 3, EINVAL);
5825
5826         strcpy(data->jmp_label, tokens[1]);
5827
5828         a = action_find(p, tokens[2]);
5829         CHECK(a, EINVAL);
5830
5831         instr->type = INSTR_JMP_ACTION_HIT;
5832         instr->jmp.ip = NULL; /* Resolved later. */
5833         instr->jmp.action_id = a->id;
5834         return 0;
5835 }
5836
5837 static int
5838 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5839                                 struct action *action,
5840                                 char **tokens,
5841                                 int n_tokens,
5842                                 struct instruction *instr,
5843                                 struct instruction_data *data)
5844 {
5845         struct action *a;
5846
5847         CHECK(!action, EINVAL);
5848         CHECK(n_tokens == 3, EINVAL);
5849
5850         strcpy(data->jmp_label, tokens[1]);
5851
5852         a = action_find(p, tokens[2]);
5853         CHECK(a, EINVAL);
5854
5855         instr->type = INSTR_JMP_ACTION_MISS;
5856         instr->jmp.ip = NULL; /* Resolved later. */
5857         instr->jmp.action_id = a->id;
5858         return 0;
5859 }
5860
5861 static int
5862 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5863                        struct action *action,
5864                        char **tokens,
5865                        int n_tokens,
5866                        struct instruction *instr,
5867                        struct instruction_data *data)
5868 {
5869         char *a = tokens[2], *b = tokens[3];
5870         struct field *fa, *fb;
5871         uint64_t b_val;
5872         uint32_t a_struct_id, b_struct_id;
5873
5874         CHECK(n_tokens == 4, EINVAL);
5875
5876         strcpy(data->jmp_label, tokens[1]);
5877
5878         fa = struct_field_parse(p, action, a, &a_struct_id);
5879         CHECK(fa, EINVAL);
5880         CHECK(!fa->var_size, EINVAL);
5881
5882         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5883         fb = struct_field_parse(p, action, b, &b_struct_id);
5884         if (fb) {
5885                 CHECK(!fb->var_size, EINVAL);
5886
5887                 instr->type = INSTR_JMP_EQ;
5888                 if (a[0] != 'h' && b[0] == 'h')
5889                         instr->type = INSTR_JMP_EQ_MH;
5890                 if (a[0] == 'h' && b[0] != 'h')
5891                         instr->type = INSTR_JMP_EQ_HM;
5892                 if (a[0] == 'h' && b[0] == 'h')
5893                         instr->type = INSTR_JMP_EQ_HH;
5894                 instr->jmp.ip = NULL; /* Resolved later. */
5895
5896                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5897                 instr->jmp.a.n_bits = fa->n_bits;
5898                 instr->jmp.a.offset = fa->offset / 8;
5899                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5900                 instr->jmp.b.n_bits = fb->n_bits;
5901                 instr->jmp.b.offset = fb->offset / 8;
5902                 return 0;
5903         }
5904
5905         /* JMP_EQ_I. */
5906         b_val = strtoull(b, &b, 0);
5907         CHECK(!b[0], EINVAL);
5908
5909         if (a[0] == 'h')
5910                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5911
5912         instr->type = INSTR_JMP_EQ_I;
5913         instr->jmp.ip = NULL; /* Resolved later. */
5914         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5915         instr->jmp.a.n_bits = fa->n_bits;
5916         instr->jmp.a.offset = fa->offset / 8;
5917         instr->jmp.b_val = b_val;
5918         return 0;
5919 }
5920
5921 static int
5922 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5923                         struct action *action,
5924                         char **tokens,
5925                         int n_tokens,
5926                         struct instruction *instr,
5927                         struct instruction_data *data)
5928 {
5929         char *a = tokens[2], *b = tokens[3];
5930         struct field *fa, *fb;
5931         uint64_t b_val;
5932         uint32_t a_struct_id, b_struct_id;
5933
5934         CHECK(n_tokens == 4, EINVAL);
5935
5936         strcpy(data->jmp_label, tokens[1]);
5937
5938         fa = struct_field_parse(p, action, a, &a_struct_id);
5939         CHECK(fa, EINVAL);
5940         CHECK(!fa->var_size, EINVAL);
5941
5942         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5943         fb = struct_field_parse(p, action, b, &b_struct_id);
5944         if (fb) {
5945                 CHECK(!fb->var_size, EINVAL);
5946
5947                 instr->type = INSTR_JMP_NEQ;
5948                 if (a[0] != 'h' && b[0] == 'h')
5949                         instr->type = INSTR_JMP_NEQ_MH;
5950                 if (a[0] == 'h' && b[0] != 'h')
5951                         instr->type = INSTR_JMP_NEQ_HM;
5952                 if (a[0] == 'h' && b[0] == 'h')
5953                         instr->type = INSTR_JMP_NEQ_HH;
5954                 instr->jmp.ip = NULL; /* Resolved later. */
5955
5956                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5957                 instr->jmp.a.n_bits = fa->n_bits;
5958                 instr->jmp.a.offset = fa->offset / 8;
5959                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5960                 instr->jmp.b.n_bits = fb->n_bits;
5961                 instr->jmp.b.offset = fb->offset / 8;
5962                 return 0;
5963         }
5964
5965         /* JMP_NEQ_I. */
5966         b_val = strtoull(b, &b, 0);
5967         CHECK(!b[0], EINVAL);
5968
5969         if (a[0] == 'h')
5970                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5971
5972         instr->type = INSTR_JMP_NEQ_I;
5973         instr->jmp.ip = NULL; /* Resolved later. */
5974         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5975         instr->jmp.a.n_bits = fa->n_bits;
5976         instr->jmp.a.offset = fa->offset / 8;
5977         instr->jmp.b_val = b_val;
5978         return 0;
5979 }
5980
5981 static int
5982 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5983                        struct action *action,
5984                        char **tokens,
5985                        int n_tokens,
5986                        struct instruction *instr,
5987                        struct instruction_data *data)
5988 {
5989         char *a = tokens[2], *b = tokens[3];
5990         struct field *fa, *fb;
5991         uint64_t b_val;
5992         uint32_t a_struct_id, b_struct_id;
5993
5994         CHECK(n_tokens == 4, EINVAL);
5995
5996         strcpy(data->jmp_label, tokens[1]);
5997
5998         fa = struct_field_parse(p, action, a, &a_struct_id);
5999         CHECK(fa, EINVAL);
6000         CHECK(!fa->var_size, EINVAL);
6001
6002         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6003         fb = struct_field_parse(p, action, b, &b_struct_id);
6004         if (fb) {
6005                 CHECK(!fb->var_size, EINVAL);
6006
6007                 instr->type = INSTR_JMP_LT;
6008                 if (a[0] == 'h' && b[0] != 'h')
6009                         instr->type = INSTR_JMP_LT_HM;
6010                 if (a[0] != 'h' && b[0] == 'h')
6011                         instr->type = INSTR_JMP_LT_MH;
6012                 if (a[0] == 'h' && b[0] == 'h')
6013                         instr->type = INSTR_JMP_LT_HH;
6014                 instr->jmp.ip = NULL; /* Resolved later. */
6015
6016                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6017                 instr->jmp.a.n_bits = fa->n_bits;
6018                 instr->jmp.a.offset = fa->offset / 8;
6019                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6020                 instr->jmp.b.n_bits = fb->n_bits;
6021                 instr->jmp.b.offset = fb->offset / 8;
6022                 return 0;
6023         }
6024
6025         /* JMP_LT_MI, JMP_LT_HI. */
6026         b_val = strtoull(b, &b, 0);
6027         CHECK(!b[0], EINVAL);
6028
6029         instr->type = INSTR_JMP_LT_MI;
6030         if (a[0] == 'h')
6031                 instr->type = INSTR_JMP_LT_HI;
6032         instr->jmp.ip = NULL; /* Resolved later. */
6033
6034         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6035         instr->jmp.a.n_bits = fa->n_bits;
6036         instr->jmp.a.offset = fa->offset / 8;
6037         instr->jmp.b_val = b_val;
6038         return 0;
6039 }
6040
6041 static int
6042 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6043                        struct action *action,
6044                        char **tokens,
6045                        int n_tokens,
6046                        struct instruction *instr,
6047                        struct instruction_data *data)
6048 {
6049         char *a = tokens[2], *b = tokens[3];
6050         struct field *fa, *fb;
6051         uint64_t b_val;
6052         uint32_t a_struct_id, b_struct_id;
6053
6054         CHECK(n_tokens == 4, EINVAL);
6055
6056         strcpy(data->jmp_label, tokens[1]);
6057
6058         fa = struct_field_parse(p, action, a, &a_struct_id);
6059         CHECK(fa, EINVAL);
6060         CHECK(!fa->var_size, EINVAL);
6061
6062         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6063         fb = struct_field_parse(p, action, b, &b_struct_id);
6064         if (fb) {
6065                 CHECK(!fb->var_size, EINVAL);
6066
6067                 instr->type = INSTR_JMP_GT;
6068                 if (a[0] == 'h' && b[0] != 'h')
6069                         instr->type = INSTR_JMP_GT_HM;
6070                 if (a[0] != 'h' && b[0] == 'h')
6071                         instr->type = INSTR_JMP_GT_MH;
6072                 if (a[0] == 'h' && b[0] == 'h')
6073                         instr->type = INSTR_JMP_GT_HH;
6074                 instr->jmp.ip = NULL; /* Resolved later. */
6075
6076                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6077                 instr->jmp.a.n_bits = fa->n_bits;
6078                 instr->jmp.a.offset = fa->offset / 8;
6079                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6080                 instr->jmp.b.n_bits = fb->n_bits;
6081                 instr->jmp.b.offset = fb->offset / 8;
6082                 return 0;
6083         }
6084
6085         /* JMP_GT_MI, JMP_GT_HI. */
6086         b_val = strtoull(b, &b, 0);
6087         CHECK(!b[0], EINVAL);
6088
6089         instr->type = INSTR_JMP_GT_MI;
6090         if (a[0] == 'h')
6091                 instr->type = INSTR_JMP_GT_HI;
6092         instr->jmp.ip = NULL; /* Resolved later. */
6093
6094         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6095         instr->jmp.a.n_bits = fa->n_bits;
6096         instr->jmp.a.offset = fa->offset / 8;
6097         instr->jmp.b_val = b_val;
6098         return 0;
6099 }
6100
6101 static inline void
6102 instr_jmp_exec(struct rte_swx_pipeline *p)
6103 {
6104         struct thread *t = &p->threads[p->thread_id];
6105         struct instruction *ip = t->ip;
6106
6107         TRACE("[Thread %2u] jmp\n", p->thread_id);
6108
6109         thread_ip_set(t, ip->jmp.ip);
6110 }
6111
6112 static inline void
6113 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6114 {
6115         struct thread *t = &p->threads[p->thread_id];
6116         struct instruction *ip = t->ip;
6117         uint32_t header_id = ip->jmp.header_id;
6118
6119         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6120
6121         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6122 }
6123
6124 static inline void
6125 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6126 {
6127         struct thread *t = &p->threads[p->thread_id];
6128         struct instruction *ip = t->ip;
6129         uint32_t header_id = ip->jmp.header_id;
6130
6131         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6132
6133         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6134 }
6135
6136 static inline void
6137 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6138 {
6139         struct thread *t = &p->threads[p->thread_id];
6140         struct instruction *ip = t->ip;
6141         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6142
6143         TRACE("[Thread %2u] jmph\n", p->thread_id);
6144
6145         t->ip = ip_next[t->hit];
6146 }
6147
6148 static inline void
6149 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6150 {
6151         struct thread *t = &p->threads[p->thread_id];
6152         struct instruction *ip = t->ip;
6153         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6154
6155         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6156
6157         t->ip = ip_next[t->hit];
6158 }
6159
6160 static inline void
6161 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6162 {
6163         struct thread *t = &p->threads[p->thread_id];
6164         struct instruction *ip = t->ip;
6165
6166         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6167
6168         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6169 }
6170
6171 static inline void
6172 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6173 {
6174         struct thread *t = &p->threads[p->thread_id];
6175         struct instruction *ip = t->ip;
6176
6177         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6178
6179         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6180 }
6181
6182 static inline void
6183 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6184 {
6185         struct thread *t = &p->threads[p->thread_id];
6186         struct instruction *ip = t->ip;
6187
6188         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6189
6190         JMP_CMP(t, ip, ==);
6191 }
6192
6193 static inline void
6194 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6195 {
6196         struct thread *t = &p->threads[p->thread_id];
6197         struct instruction *ip = t->ip;
6198
6199         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6200
6201         JMP_CMP_MH(t, ip, ==);
6202 }
6203
6204 static inline void
6205 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6206 {
6207         struct thread *t = &p->threads[p->thread_id];
6208         struct instruction *ip = t->ip;
6209
6210         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6211
6212         JMP_CMP_HM(t, ip, ==);
6213 }
6214
6215 static inline void
6216 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6217 {
6218         struct thread *t = &p->threads[p->thread_id];
6219         struct instruction *ip = t->ip;
6220
6221         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6222
6223         JMP_CMP_HH_FAST(t, ip, ==);
6224 }
6225
6226 static inline void
6227 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6228 {
6229         struct thread *t = &p->threads[p->thread_id];
6230         struct instruction *ip = t->ip;
6231
6232         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6233
6234         JMP_CMP_I(t, ip, ==);
6235 }
6236
6237 static inline void
6238 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6239 {
6240         struct thread *t = &p->threads[p->thread_id];
6241         struct instruction *ip = t->ip;
6242
6243         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6244
6245         JMP_CMP(t, ip, !=);
6246 }
6247
6248 static inline void
6249 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6250 {
6251         struct thread *t = &p->threads[p->thread_id];
6252         struct instruction *ip = t->ip;
6253
6254         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6255
6256         JMP_CMP_MH(t, ip, !=);
6257 }
6258
6259 static inline void
6260 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6261 {
6262         struct thread *t = &p->threads[p->thread_id];
6263         struct instruction *ip = t->ip;
6264
6265         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6266
6267         JMP_CMP_HM(t, ip, !=);
6268 }
6269
6270 static inline void
6271 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6272 {
6273         struct thread *t = &p->threads[p->thread_id];
6274         struct instruction *ip = t->ip;
6275
6276         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6277
6278         JMP_CMP_HH_FAST(t, ip, !=);
6279 }
6280
6281 static inline void
6282 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6283 {
6284         struct thread *t = &p->threads[p->thread_id];
6285         struct instruction *ip = t->ip;
6286
6287         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6288
6289         JMP_CMP_I(t, ip, !=);
6290 }
6291
6292 static inline void
6293 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6294 {
6295         struct thread *t = &p->threads[p->thread_id];
6296         struct instruction *ip = t->ip;
6297
6298         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6299
6300         JMP_CMP(t, ip, <);
6301 }
6302
6303 static inline void
6304 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6305 {
6306         struct thread *t = &p->threads[p->thread_id];
6307         struct instruction *ip = t->ip;
6308
6309         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6310
6311         JMP_CMP_MH(t, ip, <);
6312 }
6313
6314 static inline void
6315 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6316 {
6317         struct thread *t = &p->threads[p->thread_id];
6318         struct instruction *ip = t->ip;
6319
6320         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6321
6322         JMP_CMP_HM(t, ip, <);
6323 }
6324
6325 static inline void
6326 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6327 {
6328         struct thread *t = &p->threads[p->thread_id];
6329         struct instruction *ip = t->ip;
6330
6331         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6332
6333         JMP_CMP_HH(t, ip, <);
6334 }
6335
6336 static inline void
6337 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6338 {
6339         struct thread *t = &p->threads[p->thread_id];
6340         struct instruction *ip = t->ip;
6341
6342         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6343
6344         JMP_CMP_MI(t, ip, <);
6345 }
6346
6347 static inline void
6348 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6349 {
6350         struct thread *t = &p->threads[p->thread_id];
6351         struct instruction *ip = t->ip;
6352
6353         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6354
6355         JMP_CMP_HI(t, ip, <);
6356 }
6357
6358 static inline void
6359 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6360 {
6361         struct thread *t = &p->threads[p->thread_id];
6362         struct instruction *ip = t->ip;
6363
6364         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6365
6366         JMP_CMP(t, ip, >);
6367 }
6368
6369 static inline void
6370 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6371 {
6372         struct thread *t = &p->threads[p->thread_id];
6373         struct instruction *ip = t->ip;
6374
6375         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6376
6377         JMP_CMP_MH(t, ip, >);
6378 }
6379
6380 static inline void
6381 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6382 {
6383         struct thread *t = &p->threads[p->thread_id];
6384         struct instruction *ip = t->ip;
6385
6386         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6387
6388         JMP_CMP_HM(t, ip, >);
6389 }
6390
6391 static inline void
6392 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6393 {
6394         struct thread *t = &p->threads[p->thread_id];
6395         struct instruction *ip = t->ip;
6396
6397         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6398
6399         JMP_CMP_HH(t, ip, >);
6400 }
6401
6402 static inline void
6403 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6404 {
6405         struct thread *t = &p->threads[p->thread_id];
6406         struct instruction *ip = t->ip;
6407
6408         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6409
6410         JMP_CMP_MI(t, ip, >);
6411 }
6412
6413 static inline void
6414 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6415 {
6416         struct thread *t = &p->threads[p->thread_id];
6417         struct instruction *ip = t->ip;
6418
6419         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6420
6421         JMP_CMP_HI(t, ip, >);
6422 }
6423
6424 /*
6425  * return.
6426  */
6427 static int
6428 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6429                        struct action *action,
6430                        char **tokens __rte_unused,
6431                        int n_tokens,
6432                        struct instruction *instr,
6433                        struct instruction_data *data __rte_unused)
6434 {
6435         CHECK(action, EINVAL);
6436         CHECK(n_tokens == 1, EINVAL);
6437
6438         instr->type = INSTR_RETURN;
6439         return 0;
6440 }
6441
6442 static inline void
6443 instr_return_exec(struct rte_swx_pipeline *p)
6444 {
6445         struct thread *t = &p->threads[p->thread_id];
6446
6447         TRACE("[Thread %2u] return\n", p->thread_id);
6448
6449         t->ip = t->ret;
6450 }
6451
6452 static int
6453 instr_translate(struct rte_swx_pipeline *p,
6454                 struct action *action,
6455                 char *string,
6456                 struct instruction *instr,
6457                 struct instruction_data *data)
6458 {
6459         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6460         int n_tokens = 0, tpos = 0;
6461
6462         /* Parse the instruction string into tokens. */
6463         for ( ; ; ) {
6464                 char *token;
6465
6466                 token = strtok_r(string, " \t\v", &string);
6467                 if (!token)
6468                         break;
6469
6470                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6471                 CHECK_NAME(token, EINVAL);
6472
6473                 tokens[n_tokens] = token;
6474                 n_tokens++;
6475         }
6476
6477         CHECK(n_tokens, EINVAL);
6478
6479         /* Handle the optional instruction label. */
6480         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6481                 strcpy(data->label, tokens[0]);
6482
6483                 tpos += 2;
6484                 CHECK(n_tokens - tpos, EINVAL);
6485         }
6486
6487         /* Identify the instruction type. */
6488         if (!strcmp(tokens[tpos], "rx"))
6489                 return instr_rx_translate(p,
6490                                           action,
6491                                           &tokens[tpos],
6492                                           n_tokens - tpos,
6493                                           instr,
6494                                           data);
6495
6496         if (!strcmp(tokens[tpos], "tx"))
6497                 return instr_tx_translate(p,
6498                                           action,
6499                                           &tokens[tpos],
6500                                           n_tokens - tpos,
6501                                           instr,
6502                                           data);
6503
6504         if (!strcmp(tokens[tpos], "drop"))
6505                 return instr_drop_translate(p,
6506                                             action,
6507                                             &tokens[tpos],
6508                                             n_tokens - tpos,
6509                                             instr,
6510                                             data);
6511
6512         if (!strcmp(tokens[tpos], "extract"))
6513                 return instr_hdr_extract_translate(p,
6514                                                    action,
6515                                                    &tokens[tpos],
6516                                                    n_tokens - tpos,
6517                                                    instr,
6518                                                    data);
6519
6520         if (!strcmp(tokens[tpos], "lookahead"))
6521                 return instr_hdr_lookahead_translate(p,
6522                                                      action,
6523                                                      &tokens[tpos],
6524                                                      n_tokens - tpos,
6525                                                      instr,
6526                                                      data);
6527
6528         if (!strcmp(tokens[tpos], "emit"))
6529                 return instr_hdr_emit_translate(p,
6530                                                 action,
6531                                                 &tokens[tpos],
6532                                                 n_tokens - tpos,
6533                                                 instr,
6534                                                 data);
6535
6536         if (!strcmp(tokens[tpos], "validate"))
6537                 return instr_hdr_validate_translate(p,
6538                                                     action,
6539                                                     &tokens[tpos],
6540                                                     n_tokens - tpos,
6541                                                     instr,
6542                                                     data);
6543
6544         if (!strcmp(tokens[tpos], "invalidate"))
6545                 return instr_hdr_invalidate_translate(p,
6546                                                       action,
6547                                                       &tokens[tpos],
6548                                                       n_tokens - tpos,
6549                                                       instr,
6550                                                       data);
6551
6552         if (!strcmp(tokens[tpos], "mov"))
6553                 return instr_mov_translate(p,
6554                                            action,
6555                                            &tokens[tpos],
6556                                            n_tokens - tpos,
6557                                            instr,
6558                                            data);
6559
6560         if (!strcmp(tokens[tpos], "add"))
6561                 return instr_alu_add_translate(p,
6562                                                action,
6563                                                &tokens[tpos],
6564                                                n_tokens - tpos,
6565                                                instr,
6566                                                data);
6567
6568         if (!strcmp(tokens[tpos], "sub"))
6569                 return instr_alu_sub_translate(p,
6570                                                action,
6571                                                &tokens[tpos],
6572                                                n_tokens - tpos,
6573                                                instr,
6574                                                data);
6575
6576         if (!strcmp(tokens[tpos], "ckadd"))
6577                 return instr_alu_ckadd_translate(p,
6578                                                  action,
6579                                                  &tokens[tpos],
6580                                                  n_tokens - tpos,
6581                                                  instr,
6582                                                  data);
6583
6584         if (!strcmp(tokens[tpos], "cksub"))
6585                 return instr_alu_cksub_translate(p,
6586                                                  action,
6587                                                  &tokens[tpos],
6588                                                  n_tokens - tpos,
6589                                                  instr,
6590                                                  data);
6591
6592         if (!strcmp(tokens[tpos], "and"))
6593                 return instr_alu_and_translate(p,
6594                                                action,
6595                                                &tokens[tpos],
6596                                                n_tokens - tpos,
6597                                                instr,
6598                                                data);
6599
6600         if (!strcmp(tokens[tpos], "or"))
6601                 return instr_alu_or_translate(p,
6602                                               action,
6603                                               &tokens[tpos],
6604                                               n_tokens - tpos,
6605                                               instr,
6606                                               data);
6607
6608         if (!strcmp(tokens[tpos], "xor"))
6609                 return instr_alu_xor_translate(p,
6610                                                action,
6611                                                &tokens[tpos],
6612                                                n_tokens - tpos,
6613                                                instr,
6614                                                data);
6615
6616         if (!strcmp(tokens[tpos], "shl"))
6617                 return instr_alu_shl_translate(p,
6618                                                action,
6619                                                &tokens[tpos],
6620                                                n_tokens - tpos,
6621                                                instr,
6622                                                data);
6623
6624         if (!strcmp(tokens[tpos], "shr"))
6625                 return instr_alu_shr_translate(p,
6626                                                action,
6627                                                &tokens[tpos],
6628                                                n_tokens - tpos,
6629                                                instr,
6630                                                data);
6631
6632         if (!strcmp(tokens[tpos], "regprefetch"))
6633                 return instr_regprefetch_translate(p,
6634                                                    action,
6635                                                    &tokens[tpos],
6636                                                    n_tokens - tpos,
6637                                                    instr,
6638                                                    data);
6639
6640         if (!strcmp(tokens[tpos], "regrd"))
6641                 return instr_regrd_translate(p,
6642                                              action,
6643                                              &tokens[tpos],
6644                                              n_tokens - tpos,
6645                                              instr,
6646                                              data);
6647
6648         if (!strcmp(tokens[tpos], "regwr"))
6649                 return instr_regwr_translate(p,
6650                                              action,
6651                                              &tokens[tpos],
6652                                              n_tokens - tpos,
6653                                              instr,
6654                                              data);
6655
6656         if (!strcmp(tokens[tpos], "regadd"))
6657                 return instr_regadd_translate(p,
6658                                               action,
6659                                               &tokens[tpos],
6660                                               n_tokens - tpos,
6661                                               instr,
6662                                               data);
6663
6664         if (!strcmp(tokens[tpos], "metprefetch"))
6665                 return instr_metprefetch_translate(p,
6666                                                    action,
6667                                                    &tokens[tpos],
6668                                                    n_tokens - tpos,
6669                                                    instr,
6670                                                    data);
6671
6672         if (!strcmp(tokens[tpos], "meter"))
6673                 return instr_meter_translate(p,
6674                                              action,
6675                                              &tokens[tpos],
6676                                              n_tokens - tpos,
6677                                              instr,
6678                                              data);
6679
6680         if (!strcmp(tokens[tpos], "table"))
6681                 return instr_table_translate(p,
6682                                              action,
6683                                              &tokens[tpos],
6684                                              n_tokens - tpos,
6685                                              instr,
6686                                              data);
6687
6688         if (!strcmp(tokens[tpos], "learn"))
6689                 return instr_learn_translate(p,
6690                                              action,
6691                                              &tokens[tpos],
6692                                              n_tokens - tpos,
6693                                              instr,
6694                                              data);
6695
6696         if (!strcmp(tokens[tpos], "forget"))
6697                 return instr_forget_translate(p,
6698                                               action,
6699                                               &tokens[tpos],
6700                                               n_tokens - tpos,
6701                                               instr,
6702                                               data);
6703
6704         if (!strcmp(tokens[tpos], "extern"))
6705                 return instr_extern_translate(p,
6706                                               action,
6707                                               &tokens[tpos],
6708                                               n_tokens - tpos,
6709                                               instr,
6710                                               data);
6711
6712         if (!strcmp(tokens[tpos], "jmp"))
6713                 return instr_jmp_translate(p,
6714                                            action,
6715                                            &tokens[tpos],
6716                                            n_tokens - tpos,
6717                                            instr,
6718                                            data);
6719
6720         if (!strcmp(tokens[tpos], "jmpv"))
6721                 return instr_jmp_valid_translate(p,
6722                                                  action,
6723                                                  &tokens[tpos],
6724                                                  n_tokens - tpos,
6725                                                  instr,
6726                                                  data);
6727
6728         if (!strcmp(tokens[tpos], "jmpnv"))
6729                 return instr_jmp_invalid_translate(p,
6730                                                    action,
6731                                                    &tokens[tpos],
6732                                                    n_tokens - tpos,
6733                                                    instr,
6734                                                    data);
6735
6736         if (!strcmp(tokens[tpos], "jmph"))
6737                 return instr_jmp_hit_translate(p,
6738                                                action,
6739                                                &tokens[tpos],
6740                                                n_tokens - tpos,
6741                                                instr,
6742                                                data);
6743
6744         if (!strcmp(tokens[tpos], "jmpnh"))
6745                 return instr_jmp_miss_translate(p,
6746                                                 action,
6747                                                 &tokens[tpos],
6748                                                 n_tokens - tpos,
6749                                                 instr,
6750                                                 data);
6751
6752         if (!strcmp(tokens[tpos], "jmpa"))
6753                 return instr_jmp_action_hit_translate(p,
6754                                                       action,
6755                                                       &tokens[tpos],
6756                                                       n_tokens - tpos,
6757                                                       instr,
6758                                                       data);
6759
6760         if (!strcmp(tokens[tpos], "jmpna"))
6761                 return instr_jmp_action_miss_translate(p,
6762                                                        action,
6763                                                        &tokens[tpos],
6764                                                        n_tokens - tpos,
6765                                                        instr,
6766                                                        data);
6767
6768         if (!strcmp(tokens[tpos], "jmpeq"))
6769                 return instr_jmp_eq_translate(p,
6770                                               action,
6771                                               &tokens[tpos],
6772                                               n_tokens - tpos,
6773                                               instr,
6774                                               data);
6775
6776         if (!strcmp(tokens[tpos], "jmpneq"))
6777                 return instr_jmp_neq_translate(p,
6778                                                action,
6779                                                &tokens[tpos],
6780                                                n_tokens - tpos,
6781                                                instr,
6782                                                data);
6783
6784         if (!strcmp(tokens[tpos], "jmplt"))
6785                 return instr_jmp_lt_translate(p,
6786                                               action,
6787                                               &tokens[tpos],
6788                                               n_tokens - tpos,
6789                                               instr,
6790                                               data);
6791
6792         if (!strcmp(tokens[tpos], "jmpgt"))
6793                 return instr_jmp_gt_translate(p,
6794                                               action,
6795                                               &tokens[tpos],
6796                                               n_tokens - tpos,
6797                                               instr,
6798                                               data);
6799
6800         if (!strcmp(tokens[tpos], "return"))
6801                 return instr_return_translate(p,
6802                                               action,
6803                                               &tokens[tpos],
6804                                               n_tokens - tpos,
6805                                               instr,
6806                                               data);
6807
6808         CHECK(0, EINVAL);
6809 }
6810
6811 static struct instruction_data *
6812 label_find(struct instruction_data *data, uint32_t n, const char *label)
6813 {
6814         uint32_t i;
6815
6816         for (i = 0; i < n; i++)
6817                 if (!strcmp(label, data[i].label))
6818                         return &data[i];
6819
6820         return NULL;
6821 }
6822
6823 static uint32_t
6824 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6825 {
6826         uint32_t count = 0, i;
6827
6828         if (!label[0])
6829                 return 0;
6830
6831         for (i = 0; i < n; i++)
6832                 if (!strcmp(label, data[i].jmp_label))
6833                         count++;
6834
6835         return count;
6836 }
6837
6838 static int
6839 instr_label_check(struct instruction_data *instruction_data,
6840                   uint32_t n_instructions)
6841 {
6842         uint32_t i;
6843
6844         /* Check that all instruction labels are unique. */
6845         for (i = 0; i < n_instructions; i++) {
6846                 struct instruction_data *data = &instruction_data[i];
6847                 char *label = data->label;
6848                 uint32_t j;
6849
6850                 if (!label[0])
6851                         continue;
6852
6853                 for (j = i + 1; j < n_instructions; j++)
6854                         CHECK(strcmp(label, data[j].label), EINVAL);
6855         }
6856
6857         /* Get users for each instruction label. */
6858         for (i = 0; i < n_instructions; i++) {
6859                 struct instruction_data *data = &instruction_data[i];
6860                 char *label = data->label;
6861
6862                 data->n_users = label_is_used(instruction_data,
6863                                               n_instructions,
6864                                               label);
6865         }
6866
6867         return 0;
6868 }
6869
6870 static int
6871 instr_jmp_resolve(struct instruction *instructions,
6872                   struct instruction_data *instruction_data,
6873                   uint32_t n_instructions)
6874 {
6875         uint32_t i;
6876
6877         for (i = 0; i < n_instructions; i++) {
6878                 struct instruction *instr = &instructions[i];
6879                 struct instruction_data *data = &instruction_data[i];
6880                 struct instruction_data *found;
6881
6882                 if (!instruction_is_jmp(instr))
6883                         continue;
6884
6885                 found = label_find(instruction_data,
6886                                    n_instructions,
6887                                    data->jmp_label);
6888                 CHECK(found, EINVAL);
6889
6890                 instr->jmp.ip = &instructions[found - instruction_data];
6891         }
6892
6893         return 0;
6894 }
6895
6896 static int
6897 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6898              struct action *a,
6899              struct instruction *instr,
6900              struct instruction_data *data __rte_unused,
6901              uint32_t n_instructions)
6902 {
6903         if (!a) {
6904                 enum instruction_type type;
6905                 uint32_t i;
6906
6907                 /* Check that the first instruction is rx. */
6908                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6909
6910                 /* Check that there is at least one tx instruction. */
6911                 for (i = 0; i < n_instructions; i++) {
6912                         type = instr[i].type;
6913
6914                         if (instruction_is_tx(type))
6915                                 break;
6916                 }
6917                 CHECK(i < n_instructions, EINVAL);
6918
6919                 /* Check that the last instruction is either tx or unconditional
6920                  * jump.
6921                  */
6922                 type = instr[n_instructions - 1].type;
6923                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6924         }
6925
6926         if (a) {
6927                 enum instruction_type type;
6928                 uint32_t i;
6929
6930                 /* Check that there is at least one return or tx instruction. */
6931                 for (i = 0; i < n_instructions; i++) {
6932                         type = instr[i].type;
6933
6934                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6935                                 break;
6936                 }
6937                 CHECK(i < n_instructions, EINVAL);
6938         }
6939
6940         return 0;
6941 }
6942
6943 static uint32_t
6944 instr_compact(struct instruction *instructions,
6945               struct instruction_data *instruction_data,
6946               uint32_t n_instructions)
6947 {
6948         uint32_t i, pos = 0;
6949
6950         /* Eliminate the invalid instructions that have been optimized out. */
6951         for (i = 0; i < n_instructions; i++) {
6952                 struct instruction *instr = &instructions[i];
6953                 struct instruction_data *data = &instruction_data[i];
6954
6955                 if (data->invalid)
6956                         continue;
6957
6958                 if (i != pos) {
6959                         memcpy(&instructions[pos], instr, sizeof(*instr));
6960                         memcpy(&instruction_data[pos], data, sizeof(*data));
6961                 }
6962
6963                 pos++;
6964         }
6965
6966         return pos;
6967 }
6968
6969 static int
6970 instr_pattern_extract_many_search(struct instruction *instr,
6971                                   struct instruction_data *data,
6972                                   uint32_t n_instr,
6973                                   uint32_t *n_pattern_instr)
6974 {
6975         uint32_t i;
6976
6977         for (i = 0; i < n_instr; i++) {
6978                 if (data[i].invalid)
6979                         break;
6980
6981                 if (instr[i].type != INSTR_HDR_EXTRACT)
6982                         break;
6983
6984                 if (i == RTE_DIM(instr->io.hdr.header_id))
6985                         break;
6986
6987                 if (i && data[i].n_users)
6988                         break;
6989         }
6990
6991         if (i < 2)
6992                 return 0;
6993
6994         *n_pattern_instr = i;
6995         return 1;
6996 }
6997
6998 static void
6999 instr_pattern_extract_many_replace(struct instruction *instr,
7000                                    struct instruction_data *data,
7001                                    uint32_t n_instr)
7002 {
7003         uint32_t i;
7004
7005         for (i = 1; i < n_instr; i++) {
7006                 instr[0].type++;
7007                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7008                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7009                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7010
7011                 data[i].invalid = 1;
7012         }
7013 }
7014
7015 static uint32_t
7016 instr_pattern_extract_many_optimize(struct instruction *instructions,
7017                                     struct instruction_data *instruction_data,
7018                                     uint32_t n_instructions)
7019 {
7020         uint32_t i;
7021
7022         for (i = 0; i < n_instructions; ) {
7023                 struct instruction *instr = &instructions[i];
7024                 struct instruction_data *data = &instruction_data[i];
7025                 uint32_t n_instr = 0;
7026                 int detected;
7027
7028                 /* Extract many. */
7029                 detected = instr_pattern_extract_many_search(instr,
7030                                                              data,
7031                                                              n_instructions - i,
7032                                                              &n_instr);
7033                 if (detected) {
7034                         instr_pattern_extract_many_replace(instr,
7035                                                            data,
7036                                                            n_instr);
7037                         i += n_instr;
7038                         continue;
7039                 }
7040
7041                 /* No pattern starting at the current instruction. */
7042                 i++;
7043         }
7044
7045         /* Eliminate the invalid instructions that have been optimized out. */
7046         n_instructions = instr_compact(instructions,
7047                                        instruction_data,
7048                                        n_instructions);
7049
7050         return n_instructions;
7051 }
7052
7053 static int
7054 instr_pattern_emit_many_tx_search(struct instruction *instr,
7055                                   struct instruction_data *data,
7056                                   uint32_t n_instr,
7057                                   uint32_t *n_pattern_instr)
7058 {
7059         uint32_t i;
7060
7061         for (i = 0; i < n_instr; i++) {
7062                 if (data[i].invalid)
7063                         break;
7064
7065                 if (instr[i].type != INSTR_HDR_EMIT)
7066                         break;
7067
7068                 if (i == RTE_DIM(instr->io.hdr.header_id))
7069                         break;
7070
7071                 if (i && data[i].n_users)
7072                         break;
7073         }
7074
7075         if (!i)
7076                 return 0;
7077
7078         if (!instruction_is_tx(instr[i].type))
7079                 return 0;
7080
7081         if (data[i].n_users)
7082                 return 0;
7083
7084         i++;
7085
7086         *n_pattern_instr = i;
7087         return 1;
7088 }
7089
7090 static void
7091 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7092                                    struct instruction_data *data,
7093                                    uint32_t n_instr)
7094 {
7095         uint32_t i;
7096
7097         /* Any emit instruction in addition to the first one. */
7098         for (i = 1; i < n_instr - 1; i++) {
7099                 instr[0].type++;
7100                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7101                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7102                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7103
7104                 data[i].invalid = 1;
7105         }
7106
7107         /* The TX instruction is the last one in the pattern. */
7108         instr[0].type++;
7109         instr[0].io.io.offset = instr[i].io.io.offset;
7110         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7111         data[i].invalid = 1;
7112 }
7113
7114 static uint32_t
7115 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7116                                     struct instruction_data *instruction_data,
7117                                     uint32_t n_instructions)
7118 {
7119         uint32_t i;
7120
7121         for (i = 0; i < n_instructions; ) {
7122                 struct instruction *instr = &instructions[i];
7123                 struct instruction_data *data = &instruction_data[i];
7124                 uint32_t n_instr = 0;
7125                 int detected;
7126
7127                 /* Emit many + TX. */
7128                 detected = instr_pattern_emit_many_tx_search(instr,
7129                                                              data,
7130                                                              n_instructions - i,
7131                                                              &n_instr);
7132                 if (detected) {
7133                         instr_pattern_emit_many_tx_replace(instr,
7134                                                            data,
7135                                                            n_instr);
7136                         i += n_instr;
7137                         continue;
7138                 }
7139
7140                 /* No pattern starting at the current instruction. */
7141                 i++;
7142         }
7143
7144         /* Eliminate the invalid instructions that have been optimized out. */
7145         n_instructions = instr_compact(instructions,
7146                                        instruction_data,
7147                                        n_instructions);
7148
7149         return n_instructions;
7150 }
7151
7152 static uint32_t
7153 action_arg_src_mov_count(struct action *a,
7154                          uint32_t arg_id,
7155                          struct instruction *instructions,
7156                          struct instruction_data *instruction_data,
7157                          uint32_t n_instructions);
7158
7159 static int
7160 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7161                                       struct action *a,
7162                                       struct instruction *instr,
7163                                       struct instruction_data *data,
7164                                       uint32_t n_instr,
7165                                       struct instruction *instructions,
7166                                       struct instruction_data *instruction_data,
7167                                       uint32_t n_instructions,
7168                                       uint32_t *n_pattern_instr)
7169 {
7170         struct header *h;
7171         uint32_t src_field_id, i, j;
7172
7173         /* Prerequisites. */
7174         if (!a || !a->st)
7175                 return 0;
7176
7177         /* First instruction: MOV_HM. */
7178         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7179                 return 0;
7180
7181         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7182         if (!h || h->st->var_size)
7183                 return 0;
7184
7185         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7186                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7187                         break;
7188
7189         if (src_field_id == a->st->n_fields)
7190                 return 0;
7191
7192         if (instr[0].mov.dst.offset ||
7193             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7194             instr[0].mov.src.struct_id ||
7195             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7196             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7197                 return 0;
7198
7199         if ((n_instr < h->st->n_fields + 1) ||
7200              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7201                 return 0;
7202
7203         /* Subsequent instructions: MOV_HM. */
7204         for (i = 1; i < h->st->n_fields; i++)
7205                 if (data[i].invalid ||
7206                     data[i].n_users ||
7207                     (instr[i].type != INSTR_MOV_HM) ||
7208                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7209                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7210                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7211                     instr[i].mov.src.struct_id ||
7212                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7213                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7214                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7215                         return 0;
7216
7217         /* Last instruction: HDR_VALIDATE. */
7218         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7219             (instr[i].valid.header_id != h->id))
7220                 return 0;
7221
7222         /* Check that none of the action args that are used as source for this
7223          * DMA transfer are not used as source in any other mov instruction.
7224          */
7225         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7226                 uint32_t n_users;
7227
7228                 n_users = action_arg_src_mov_count(a,
7229                                                    j,
7230                                                    instructions,
7231                                                    instruction_data,
7232                                                    n_instructions);
7233                 if (n_users > 1)
7234                         return 0;
7235         }
7236
7237         *n_pattern_instr = 1 + i;
7238         return 1;
7239 }
7240
7241 static void
7242 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7243                                        struct action *a,
7244                                        struct instruction *instr,
7245                                        struct instruction_data *data,
7246                                        uint32_t n_instr)
7247 {
7248         struct header *h;
7249         uint32_t src_field_id, src_offset, i;
7250
7251         /* Read from the instructions before they are modified. */
7252         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7253         if (!h)
7254                 return;
7255
7256         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7257                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7258                         break;
7259
7260         if (src_field_id == a->st->n_fields)
7261                 return;
7262
7263         src_offset = instr[0].mov.src.offset;
7264
7265         /* Modify the instructions. */
7266         instr[0].type = INSTR_DMA_HT;
7267         instr[0].dma.dst.header_id[0] = h->id;
7268         instr[0].dma.dst.struct_id[0] = h->struct_id;
7269         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7270         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7271
7272         for (i = 1; i < n_instr; i++)
7273                 data[i].invalid = 1;
7274
7275         /* Update the endianness of the action arguments to header endianness. */
7276         for (i = 0; i < h->st->n_fields; i++)
7277                 a->args_endianness[src_field_id + i] = 1;
7278 }
7279
7280 static uint32_t
7281 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7282                                         struct action *a,
7283                                         struct instruction *instructions,
7284                                         struct instruction_data *instruction_data,
7285                                         uint32_t n_instructions)
7286 {
7287         uint32_t i;
7288
7289         if (!a || !a->st)
7290                 return n_instructions;
7291
7292         for (i = 0; i < n_instructions; ) {
7293                 struct instruction *instr = &instructions[i];
7294                 struct instruction_data *data = &instruction_data[i];
7295                 uint32_t n_instr = 0;
7296                 int detected;
7297
7298                 /* Mov all + validate. */
7299                 detected = instr_pattern_mov_all_validate_search(p,
7300                                                                  a,
7301                                                                  instr,
7302                                                                  data,
7303                                                                  n_instructions - i,
7304                                                                  instructions,
7305                                                                  instruction_data,
7306                                                                  n_instructions,
7307                                                                  &n_instr);
7308                 if (detected) {
7309                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7310                         i += n_instr;
7311                         continue;
7312                 }
7313
7314                 /* No pattern starting at the current instruction. */
7315                 i++;
7316         }
7317
7318         /* Eliminate the invalid instructions that have been optimized out. */
7319         n_instructions = instr_compact(instructions,
7320                                        instruction_data,
7321                                        n_instructions);
7322
7323         return n_instructions;
7324 }
7325
7326 static int
7327 instr_pattern_dma_many_search(struct instruction *instr,
7328                               struct instruction_data *data,
7329                               uint32_t n_instr,
7330                               uint32_t *n_pattern_instr)
7331 {
7332         uint32_t i;
7333
7334         for (i = 0; i < n_instr; i++) {
7335                 if (data[i].invalid)
7336                         break;
7337
7338                 if (instr[i].type != INSTR_DMA_HT)
7339                         break;
7340
7341                 if (i == RTE_DIM(instr->dma.dst.header_id))
7342                         break;
7343
7344                 if (i && data[i].n_users)
7345                         break;
7346         }
7347
7348         if (i < 2)
7349                 return 0;
7350
7351         *n_pattern_instr = i;
7352         return 1;
7353 }
7354
7355 static void
7356 instr_pattern_dma_many_replace(struct instruction *instr,
7357                                struct instruction_data *data,
7358                                uint32_t n_instr)
7359 {
7360         uint32_t i;
7361
7362         for (i = 1; i < n_instr; i++) {
7363                 instr[0].type++;
7364                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7365                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7366                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7367                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7368
7369                 data[i].invalid = 1;
7370         }
7371 }
7372
7373 static uint32_t
7374 instr_pattern_dma_many_optimize(struct instruction *instructions,
7375                struct instruction_data *instruction_data,
7376                uint32_t n_instructions)
7377 {
7378         uint32_t i;
7379
7380         for (i = 0; i < n_instructions; ) {
7381                 struct instruction *instr = &instructions[i];
7382                 struct instruction_data *data = &instruction_data[i];
7383                 uint32_t n_instr = 0;
7384                 int detected;
7385
7386                 /* DMA many. */
7387                 detected = instr_pattern_dma_many_search(instr,
7388                                                          data,
7389                                                          n_instructions - i,
7390                                                          &n_instr);
7391                 if (detected) {
7392                         instr_pattern_dma_many_replace(instr, data, n_instr);
7393                         i += n_instr;
7394                         continue;
7395                 }
7396
7397                 /* No pattern starting at the current instruction. */
7398                 i++;
7399         }
7400
7401         /* Eliminate the invalid instructions that have been optimized out. */
7402         n_instructions = instr_compact(instructions,
7403                                        instruction_data,
7404                                        n_instructions);
7405
7406         return n_instructions;
7407 }
7408
7409 static uint32_t
7410 instr_optimize(struct rte_swx_pipeline *p,
7411                struct action *a,
7412                struct instruction *instructions,
7413                struct instruction_data *instruction_data,
7414                uint32_t n_instructions)
7415 {
7416         /* Extract many. */
7417         n_instructions = instr_pattern_extract_many_optimize(instructions,
7418                                                              instruction_data,
7419                                                              n_instructions);
7420
7421         /* Emit many + TX. */
7422         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7423                                                              instruction_data,
7424                                                              n_instructions);
7425
7426         /* Mov all + validate. */
7427         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7428                                                                  a,
7429                                                                  instructions,
7430                                                                  instruction_data,
7431                                                                  n_instructions);
7432
7433         /* DMA many. */
7434         n_instructions = instr_pattern_dma_many_optimize(instructions,
7435                                                          instruction_data,
7436                                                          n_instructions);
7437
7438         return n_instructions;
7439 }
7440
7441 static int
7442 instruction_config(struct rte_swx_pipeline *p,
7443                    struct action *a,
7444                    const char **instructions,
7445                    uint32_t n_instructions)
7446 {
7447         struct instruction *instr = NULL;
7448         struct instruction_data *data = NULL;
7449         int err = 0;
7450         uint32_t i;
7451
7452         CHECK(n_instructions, EINVAL);
7453         CHECK(instructions, EINVAL);
7454         for (i = 0; i < n_instructions; i++)
7455                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7456
7457         /* Memory allocation. */
7458         instr = calloc(n_instructions, sizeof(struct instruction));
7459         if (!instr) {
7460                 err = -ENOMEM;
7461                 goto error;
7462         }
7463
7464         data = calloc(n_instructions, sizeof(struct instruction_data));
7465         if (!data) {
7466                 err = -ENOMEM;
7467                 goto error;
7468         }
7469
7470         for (i = 0; i < n_instructions; i++) {
7471                 char *string = strdup(instructions[i]);
7472                 if (!string) {
7473                         err = -ENOMEM;
7474                         goto error;
7475                 }
7476
7477                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7478                 if (err) {
7479                         free(string);
7480                         goto error;
7481                 }
7482
7483                 free(string);
7484         }
7485
7486         err = instr_label_check(data, n_instructions);
7487         if (err)
7488                 goto error;
7489
7490         err = instr_verify(p, a, instr, data, n_instructions);
7491         if (err)
7492                 goto error;
7493
7494         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7495
7496         err = instr_jmp_resolve(instr, data, n_instructions);
7497         if (err)
7498                 goto error;
7499
7500         if (a) {
7501                 a->instructions = instr;
7502                 a->n_instructions = n_instructions;
7503         } else {
7504                 p->instructions = instr;
7505                 p->n_instructions = n_instructions;
7506         }
7507
7508         free(data);
7509         return 0;
7510
7511 error:
7512         free(data);
7513         free(instr);
7514         return err;
7515 }
7516
7517 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7518
7519 static instr_exec_t instruction_table[] = {
7520         [INSTR_RX] = instr_rx_exec,
7521         [INSTR_TX] = instr_tx_exec,
7522         [INSTR_TX_I] = instr_tx_i_exec,
7523
7524         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7525         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7526         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7527         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7528         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7529         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7530         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7531         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7532         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7533         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7534
7535         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7536         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7537         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7538         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7539         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7540         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7541         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7542         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7543         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7544
7545         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7546         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7547
7548         [INSTR_MOV] = instr_mov_exec,
7549         [INSTR_MOV_MH] = instr_mov_mh_exec,
7550         [INSTR_MOV_HM] = instr_mov_hm_exec,
7551         [INSTR_MOV_HH] = instr_mov_hh_exec,
7552         [INSTR_MOV_I] = instr_mov_i_exec,
7553
7554         [INSTR_DMA_HT] = instr_dma_ht_exec,
7555         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7556         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7557         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7558         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7559         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7560         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7561         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7562
7563         [INSTR_ALU_ADD] = instr_alu_add_exec,
7564         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7565         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7566         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7567         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7568         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7569
7570         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7571         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7572         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7573         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7574         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7575         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7576
7577         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7578         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7579         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7580         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7581
7582         [INSTR_ALU_AND] = instr_alu_and_exec,
7583         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7584         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7585         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7586         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7587
7588         [INSTR_ALU_OR] = instr_alu_or_exec,
7589         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7590         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7591         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7592         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7593
7594         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7595         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7596         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7597         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7598         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7599
7600         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7601         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7602         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7603         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7604         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7605         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7606
7607         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7608         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7609         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7610         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7611         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7612         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7613
7614         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7615         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7616         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7617
7618         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7619         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7620         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7621         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7622         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7623         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7624
7625         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7626         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7627         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7628         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7629         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7630         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7631         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7632         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7633         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7634
7635         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7636         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7637         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7638         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7639         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7640         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7641         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7642         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7643         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7644
7645         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7646         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7647         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7648
7649         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7650         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7651         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7652         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7653         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7654         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7655         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7656         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7657         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7658         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7659         [INSTR_METER_IMM] = instr_meter_imm_exec,
7660         [INSTR_METER_IMI] = instr_meter_imi_exec,
7661
7662         [INSTR_TABLE] = instr_table_exec,
7663         [INSTR_SELECTOR] = instr_selector_exec,
7664         [INSTR_LEARNER] = instr_learner_exec,
7665         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7666         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7667         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7668         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7669
7670         [INSTR_JMP] = instr_jmp_exec,
7671         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7672         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7673         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7674         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7675         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7676         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7677
7678         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7679         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7680         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7681         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7682         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7683
7684         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7685         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7686         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7687         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7688         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7689
7690         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7691         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7692         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7693         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7694         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7695         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7696
7697         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7698         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7699         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7700         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7701         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7702         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7703
7704         [INSTR_RETURN] = instr_return_exec,
7705 };
7706
7707 static inline void
7708 instr_exec(struct rte_swx_pipeline *p)
7709 {
7710         struct thread *t = &p->threads[p->thread_id];
7711         struct instruction *ip = t->ip;
7712         instr_exec_t instr = instruction_table[ip->type];
7713
7714         instr(p);
7715 }
7716
7717 /*
7718  * Action.
7719  */
7720 static struct action *
7721 action_find(struct rte_swx_pipeline *p, const char *name)
7722 {
7723         struct action *elem;
7724
7725         if (!name)
7726                 return NULL;
7727
7728         TAILQ_FOREACH(elem, &p->actions, node)
7729                 if (strcmp(elem->name, name) == 0)
7730                         return elem;
7731
7732         return NULL;
7733 }
7734
7735 static struct action *
7736 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7737 {
7738         struct action *action = NULL;
7739
7740         TAILQ_FOREACH(action, &p->actions, node)
7741                 if (action->id == id)
7742                         return action;
7743
7744         return NULL;
7745 }
7746
7747 static struct field *
7748 action_field_find(struct action *a, const char *name)
7749 {
7750         return a->st ? struct_type_field_find(a->st, name) : NULL;
7751 }
7752
7753 static struct field *
7754 action_field_parse(struct action *action, const char *name)
7755 {
7756         if (name[0] != 't' || name[1] != '.')
7757                 return NULL;
7758
7759         return action_field_find(action, &name[2]);
7760 }
7761
7762 static int
7763 action_has_nbo_args(struct action *a)
7764 {
7765         uint32_t i;
7766
7767         /* Return if the action does not have any args. */
7768         if (!a->st)
7769                 return 0; /* FALSE */
7770
7771         for (i = 0; i < a->st->n_fields; i++)
7772                 if (a->args_endianness[i])
7773                         return 1; /* TRUE */
7774
7775         return 0; /* FALSE */
7776 }
7777
7778 static int
7779 action_does_learning(struct action *a)
7780 {
7781         uint32_t i;
7782
7783         for (i = 0; i < a->n_instructions; i++)
7784                 switch (a->instructions[i].type) {
7785                 case INSTR_LEARNER_LEARN:
7786                         return 1; /* TRUE */
7787
7788                 case INSTR_LEARNER_FORGET:
7789                         return 1; /* TRUE */
7790
7791                 default:
7792                         continue;
7793                 }
7794
7795         return 0; /* FALSE */
7796 }
7797
7798 int
7799 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7800                                const char *name,
7801                                const char *args_struct_type_name,
7802                                const char **instructions,
7803                                uint32_t n_instructions)
7804 {
7805         struct struct_type *args_struct_type = NULL;
7806         struct action *a;
7807         int err;
7808
7809         CHECK(p, EINVAL);
7810
7811         CHECK_NAME(name, EINVAL);
7812         CHECK(!action_find(p, name), EEXIST);
7813
7814         if (args_struct_type_name) {
7815                 CHECK_NAME(args_struct_type_name, EINVAL);
7816                 args_struct_type = struct_type_find(p, args_struct_type_name);
7817                 CHECK(args_struct_type, EINVAL);
7818                 CHECK(!args_struct_type->var_size, EINVAL);
7819         }
7820
7821         /* Node allocation. */
7822         a = calloc(1, sizeof(struct action));
7823         CHECK(a, ENOMEM);
7824         if (args_struct_type) {
7825                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7826                 if (!a->args_endianness) {
7827                         free(a);
7828                         CHECK(0, ENOMEM);
7829                 }
7830         }
7831
7832         /* Node initialization. */
7833         strcpy(a->name, name);
7834         a->st = args_struct_type;
7835         a->id = p->n_actions;
7836
7837         /* Instruction translation. */
7838         err = instruction_config(p, a, instructions, n_instructions);
7839         if (err) {
7840                 free(a->args_endianness);
7841                 free(a);
7842                 return err;
7843         }
7844
7845         /* Node add to tailq. */
7846         TAILQ_INSERT_TAIL(&p->actions, a, node);
7847         p->n_actions++;
7848
7849         return 0;
7850 }
7851
7852 static int
7853 action_build(struct rte_swx_pipeline *p)
7854 {
7855         struct action *action;
7856
7857         p->action_instructions = calloc(p->n_actions,
7858                                         sizeof(struct instruction *));
7859         CHECK(p->action_instructions, ENOMEM);
7860
7861         TAILQ_FOREACH(action, &p->actions, node)
7862                 p->action_instructions[action->id] = action->instructions;
7863
7864         return 0;
7865 }
7866
7867 static void
7868 action_build_free(struct rte_swx_pipeline *p)
7869 {
7870         free(p->action_instructions);
7871         p->action_instructions = NULL;
7872 }
7873
7874 static void
7875 action_free(struct rte_swx_pipeline *p)
7876 {
7877         action_build_free(p);
7878
7879         for ( ; ; ) {
7880                 struct action *action;
7881
7882                 action = TAILQ_FIRST(&p->actions);
7883                 if (!action)
7884                         break;
7885
7886                 TAILQ_REMOVE(&p->actions, action, node);
7887                 free(action->instructions);
7888                 free(action);
7889         }
7890 }
7891
7892 static uint32_t
7893 action_arg_src_mov_count(struct action *a,
7894                          uint32_t arg_id,
7895                          struct instruction *instructions,
7896                          struct instruction_data *instruction_data,
7897                          uint32_t n_instructions)
7898 {
7899         uint32_t offset, n_users = 0, i;
7900
7901         if (!a->st ||
7902             (arg_id >= a->st->n_fields) ||
7903             !instructions ||
7904             !instruction_data ||
7905             !n_instructions)
7906                 return 0;
7907
7908         offset = a->st->fields[arg_id].offset / 8;
7909
7910         for (i = 0; i < n_instructions; i++) {
7911                 struct instruction *instr = &instructions[i];
7912                 struct instruction_data *data = &instruction_data[i];
7913
7914                 if (data->invalid ||
7915                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7916                     instr->mov.src.struct_id ||
7917                     (instr->mov.src.offset != offset))
7918                         continue;
7919
7920                 n_users++;
7921         }
7922
7923         return n_users;
7924 }
7925
7926 /*
7927  * Table.
7928  */
7929 static struct table_type *
7930 table_type_find(struct rte_swx_pipeline *p, const char *name)
7931 {
7932         struct table_type *elem;
7933
7934         TAILQ_FOREACH(elem, &p->table_types, node)
7935                 if (strcmp(elem->name, name) == 0)
7936                         return elem;
7937
7938         return NULL;
7939 }
7940
7941 static struct table_type *
7942 table_type_resolve(struct rte_swx_pipeline *p,
7943                    const char *recommended_type_name,
7944                    enum rte_swx_table_match_type match_type)
7945 {
7946         struct table_type *elem;
7947
7948         /* Only consider the recommended type if the match type is correct. */
7949         if (recommended_type_name)
7950                 TAILQ_FOREACH(elem, &p->table_types, node)
7951                         if (!strcmp(elem->name, recommended_type_name) &&
7952                             (elem->match_type == match_type))
7953                                 return elem;
7954
7955         /* Ignore the recommended type and get the first element with this match
7956          * type.
7957          */
7958         TAILQ_FOREACH(elem, &p->table_types, node)
7959                 if (elem->match_type == match_type)
7960                         return elem;
7961
7962         return NULL;
7963 }
7964
7965 static struct table *
7966 table_find(struct rte_swx_pipeline *p, const char *name)
7967 {
7968         struct table *elem;
7969
7970         TAILQ_FOREACH(elem, &p->tables, node)
7971                 if (strcmp(elem->name, name) == 0)
7972                         return elem;
7973
7974         return NULL;
7975 }
7976
7977 static struct table *
7978 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7979 {
7980         struct table *table = NULL;
7981
7982         TAILQ_FOREACH(table, &p->tables, node)
7983                 if (table->id == id)
7984                         return table;
7985
7986         return NULL;
7987 }
7988
7989 int
7990 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7991                                      const char *name,
7992                                      enum rte_swx_table_match_type match_type,
7993                                      struct rte_swx_table_ops *ops)
7994 {
7995         struct table_type *elem;
7996
7997         CHECK(p, EINVAL);
7998
7999         CHECK_NAME(name, EINVAL);
8000         CHECK(!table_type_find(p, name), EEXIST);
8001
8002         CHECK(ops, EINVAL);
8003         CHECK(ops->create, EINVAL);
8004         CHECK(ops->lkp, EINVAL);
8005         CHECK(ops->free, EINVAL);
8006
8007         /* Node allocation. */
8008         elem = calloc(1, sizeof(struct table_type));
8009         CHECK(elem, ENOMEM);
8010
8011         /* Node initialization. */
8012         strcpy(elem->name, name);
8013         elem->match_type = match_type;
8014         memcpy(&elem->ops, ops, sizeof(*ops));
8015
8016         /* Node add to tailq. */
8017         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8018
8019         return 0;
8020 }
8021
8022 static int
8023 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8024                          uint32_t n_fields,
8025                          enum rte_swx_table_match_type *match_type)
8026 {
8027         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
8028
8029         for (i = 0; i < n_fields; i++) {
8030                 struct rte_swx_match_field_params  *f = &fields[i];
8031
8032                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
8033                         n_fields_em++;
8034
8035                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
8036                         n_fields_lpm++;
8037         }
8038
8039         if ((n_fields_lpm > 1) ||
8040             (n_fields_lpm && (n_fields_em != n_fields - 1)))
8041                 return -EINVAL;
8042
8043         *match_type = (n_fields_em == n_fields) ?
8044                        RTE_SWX_TABLE_MATCH_EXACT :
8045                        RTE_SWX_TABLE_MATCH_WILDCARD;
8046
8047         return 0;
8048 }
8049
8050 static int
8051 table_match_fields_check(struct rte_swx_pipeline *p,
8052                          struct rte_swx_pipeline_table_params *params,
8053                          struct header **header)
8054 {
8055         struct header *h0 = NULL;
8056         struct field *hf, *mf;
8057         uint32_t *offset = NULL, i;
8058         int status = 0;
8059
8060         /* Return if no match fields. */
8061         if (!params->n_fields) {
8062                 if (params->fields) {
8063                         status = -EINVAL;
8064                         goto end;
8065                 }
8066
8067                 if (header)
8068                         *header = NULL;
8069
8070                 return 0;
8071         }
8072
8073         /* Memory allocation. */
8074         offset = calloc(params->n_fields, sizeof(uint32_t));
8075         if (!offset) {
8076                 status = -ENOMEM;
8077                 goto end;
8078         }
8079
8080         /* Check that all the match fields belong to either the same header or
8081          * to the meta-data.
8082          */
8083         hf = header_field_parse(p, params->fields[0].name, &h0);
8084         mf = metadata_field_parse(p, params->fields[0].name);
8085         if ((!hf && !mf) || (hf && hf->var_size)) {
8086                 status = -EINVAL;
8087                 goto end;
8088         }
8089
8090         offset[0] = h0 ? hf->offset : mf->offset;
8091
8092         for (i = 1; i < params->n_fields; i++)
8093                 if (h0) {
8094                         struct header *h;
8095
8096                         hf = header_field_parse(p, params->fields[i].name, &h);
8097                         if (!hf || (h->id != h0->id) || hf->var_size) {
8098                                 status = -EINVAL;
8099                                 goto end;
8100                         }
8101
8102                         offset[i] = hf->offset;
8103                 } else {
8104                         mf = metadata_field_parse(p, params->fields[i].name);
8105                         if (!mf) {
8106                                 status = -EINVAL;
8107                                 goto end;
8108                         }
8109
8110                         offset[i] = mf->offset;
8111                 }
8112
8113         /* Check that there are no duplicated match fields. */
8114         for (i = 0; i < params->n_fields; i++) {
8115                 uint32_t j;
8116
8117                 for (j = 0; j < i; j++)
8118                         if (offset[j] == offset[i]) {
8119                                 status = -EINVAL;
8120                                 goto end;
8121                         }
8122         }
8123
8124         /* Return. */
8125         if (header)
8126                 *header = h0;
8127
8128 end:
8129         free(offset);
8130         return status;
8131 }
8132
8133 int
8134 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8135                               const char *name,
8136                               struct rte_swx_pipeline_table_params *params,
8137                               const char *recommended_table_type_name,
8138                               const char *args,
8139                               uint32_t size)
8140 {
8141         struct table_type *type;
8142         struct table *t;
8143         struct action *default_action;
8144         struct header *header = NULL;
8145         uint32_t action_data_size_max = 0, i;
8146         int status = 0;
8147
8148         CHECK(p, EINVAL);
8149
8150         CHECK_NAME(name, EINVAL);
8151         CHECK(!table_find(p, name), EEXIST);
8152         CHECK(!selector_find(p, name), EEXIST);
8153         CHECK(!learner_find(p, name), EEXIST);
8154
8155         CHECK(params, EINVAL);
8156
8157         /* Match checks. */
8158         status = table_match_fields_check(p, params, &header);
8159         if (status)
8160                 return status;
8161
8162         /* Action checks. */
8163         CHECK(params->n_actions, EINVAL);
8164         CHECK(params->action_names, EINVAL);
8165         for (i = 0; i < params->n_actions; i++) {
8166                 const char *action_name = params->action_names[i];
8167                 struct action *a;
8168                 uint32_t action_data_size;
8169
8170                 CHECK_NAME(action_name, EINVAL);
8171
8172                 a = action_find(p, action_name);
8173                 CHECK(a, EINVAL);
8174                 CHECK(!action_does_learning(a), EINVAL);
8175
8176                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8177                 if (action_data_size > action_data_size_max)
8178                         action_data_size_max = action_data_size;
8179         }
8180
8181         CHECK_NAME(params->default_action_name, EINVAL);
8182         for (i = 0; i < p->n_actions; i++)
8183                 if (!strcmp(params->action_names[i],
8184                             params->default_action_name))
8185                         break;
8186         CHECK(i < params->n_actions, EINVAL);
8187         default_action = action_find(p, params->default_action_name);
8188         CHECK((default_action->st && params->default_action_data) ||
8189               !params->default_action_data, EINVAL);
8190
8191         /* Table type checks. */
8192         if (recommended_table_type_name)
8193                 CHECK_NAME(recommended_table_type_name, EINVAL);
8194
8195         if (params->n_fields) {
8196                 enum rte_swx_table_match_type match_type;
8197
8198                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8199                 if (status)
8200                         return status;
8201
8202                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8203                 CHECK(type, EINVAL);
8204         } else {
8205                 type = NULL;
8206         }
8207
8208         /* Memory allocation. */
8209         t = calloc(1, sizeof(struct table));
8210         CHECK(t, ENOMEM);
8211
8212         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8213         if (!t->fields) {
8214                 free(t);
8215                 CHECK(0, ENOMEM);
8216         }
8217
8218         t->actions = calloc(params->n_actions, sizeof(struct action *));
8219         if (!t->actions) {
8220                 free(t->fields);
8221                 free(t);
8222                 CHECK(0, ENOMEM);
8223         }
8224
8225         if (action_data_size_max) {
8226                 t->default_action_data = calloc(1, action_data_size_max);
8227                 if (!t->default_action_data) {
8228                         free(t->actions);
8229                         free(t->fields);
8230                         free(t);
8231                         CHECK(0, ENOMEM);
8232                 }
8233         }
8234
8235         /* Node initialization. */
8236         strcpy(t->name, name);
8237         if (args && args[0])
8238                 strcpy(t->args, args);
8239         t->type = type;
8240
8241         for (i = 0; i < params->n_fields; i++) {
8242                 struct rte_swx_match_field_params *field = &params->fields[i];
8243                 struct match_field *f = &t->fields[i];
8244
8245                 f->match_type = field->match_type;
8246                 f->field = header ?
8247                         header_field_parse(p, field->name, NULL) :
8248                         metadata_field_parse(p, field->name);
8249         }
8250         t->n_fields = params->n_fields;
8251         t->header = header;
8252
8253         for (i = 0; i < params->n_actions; i++)
8254                 t->actions[i] = action_find(p, params->action_names[i]);
8255         t->default_action = default_action;
8256         if (default_action->st)
8257                 memcpy(t->default_action_data,
8258                        params->default_action_data,
8259                        default_action->st->n_bits / 8);
8260         t->n_actions = params->n_actions;
8261         t->default_action_is_const = params->default_action_is_const;
8262         t->action_data_size_max = action_data_size_max;
8263
8264         t->size = size;
8265         t->id = p->n_tables;
8266
8267         /* Node add to tailq. */
8268         TAILQ_INSERT_TAIL(&p->tables, t, node);
8269         p->n_tables++;
8270
8271         return 0;
8272 }
8273
8274 static struct rte_swx_table_params *
8275 table_params_get(struct table *table)
8276 {
8277         struct rte_swx_table_params *params;
8278         struct field *first, *last;
8279         uint8_t *key_mask;
8280         uint32_t key_size, key_offset, action_data_size, i;
8281
8282         /* Memory allocation. */
8283         params = calloc(1, sizeof(struct rte_swx_table_params));
8284         if (!params)
8285                 return NULL;
8286
8287         /* Find first (smallest offset) and last (biggest offset) match fields. */
8288         first = table->fields[0].field;
8289         last = table->fields[0].field;
8290
8291         for (i = 0; i < table->n_fields; i++) {
8292                 struct field *f = table->fields[i].field;
8293
8294                 if (f->offset < first->offset)
8295                         first = f;
8296
8297                 if (f->offset > last->offset)
8298                         last = f;
8299         }
8300
8301         /* Key offset and size. */
8302         key_offset = first->offset / 8;
8303         key_size = (last->offset + last->n_bits - first->offset) / 8;
8304
8305         /* Memory allocation. */
8306         key_mask = calloc(1, key_size);
8307         if (!key_mask) {
8308                 free(params);
8309                 return NULL;
8310         }
8311
8312         /* Key mask. */
8313         for (i = 0; i < table->n_fields; i++) {
8314                 struct field *f = table->fields[i].field;
8315                 uint32_t start = (f->offset - first->offset) / 8;
8316                 size_t size = f->n_bits / 8;
8317
8318                 memset(&key_mask[start], 0xFF, size);
8319         }
8320
8321         /* Action data size. */
8322         action_data_size = 0;
8323         for (i = 0; i < table->n_actions; i++) {
8324                 struct action *action = table->actions[i];
8325                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8326
8327                 if (ads > action_data_size)
8328                         action_data_size = ads;
8329         }
8330
8331         /* Fill in. */
8332         params->match_type = table->type->match_type;
8333         params->key_size = key_size;
8334         params->key_offset = key_offset;
8335         params->key_mask0 = key_mask;
8336         params->action_data_size = action_data_size;
8337         params->n_keys_max = table->size;
8338
8339         return params;
8340 }
8341
8342 static void
8343 table_params_free(struct rte_swx_table_params *params)
8344 {
8345         if (!params)
8346                 return;
8347
8348         free(params->key_mask0);
8349         free(params);
8350 }
8351
8352 static int
8353 table_stub_lkp(void *table __rte_unused,
8354                void *mailbox __rte_unused,
8355                uint8_t **key __rte_unused,
8356                uint64_t *action_id __rte_unused,
8357                uint8_t **action_data __rte_unused,
8358                int *hit)
8359 {
8360         *hit = 0;
8361         return 1; /* DONE. */
8362 }
8363
8364 static int
8365 table_build(struct rte_swx_pipeline *p)
8366 {
8367         uint32_t i;
8368
8369         /* Per pipeline: table statistics. */
8370         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8371         CHECK(p->table_stats, ENOMEM);
8372
8373         for (i = 0; i < p->n_tables; i++) {
8374                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8375                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8376         }
8377
8378         /* Per thread: table runt-time. */
8379         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8380                 struct thread *t = &p->threads[i];
8381                 struct table *table;
8382
8383                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8384                 CHECK(t->tables, ENOMEM);
8385
8386                 TAILQ_FOREACH(table, &p->tables, node) {
8387                         struct table_runtime *r = &t->tables[table->id];
8388
8389                         if (table->type) {
8390                                 uint64_t size;
8391
8392                                 size = table->type->ops.mailbox_size_get();
8393
8394                                 /* r->func. */
8395                                 r->func = table->type->ops.lkp;
8396
8397                                 /* r->mailbox. */
8398                                 if (size) {
8399                                         r->mailbox = calloc(1, size);
8400                                         CHECK(r->mailbox, ENOMEM);
8401                                 }
8402
8403                                 /* r->key. */
8404                                 r->key = table->header ?
8405                                         &t->structs[table->header->struct_id] :
8406                                         &t->structs[p->metadata_struct_id];
8407                         } else {
8408                                 r->func = table_stub_lkp;
8409                         }
8410                 }
8411         }
8412
8413         return 0;
8414 }
8415
8416 static void
8417 table_build_free(struct rte_swx_pipeline *p)
8418 {
8419         uint32_t i;
8420
8421         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8422                 struct thread *t = &p->threads[i];
8423                 uint32_t j;
8424
8425                 if (!t->tables)
8426                         continue;
8427
8428                 for (j = 0; j < p->n_tables; j++) {
8429                         struct table_runtime *r = &t->tables[j];
8430
8431                         free(r->mailbox);
8432                 }
8433
8434                 free(t->tables);
8435                 t->tables = NULL;
8436         }
8437
8438         if (p->table_stats) {
8439                 for (i = 0; i < p->n_tables; i++)
8440                         free(p->table_stats[i].n_pkts_action);
8441
8442                 free(p->table_stats);
8443         }
8444 }
8445
8446 static void
8447 table_free(struct rte_swx_pipeline *p)
8448 {
8449         table_build_free(p);
8450
8451         /* Tables. */
8452         for ( ; ; ) {
8453                 struct table *elem;
8454
8455                 elem = TAILQ_FIRST(&p->tables);
8456                 if (!elem)
8457                         break;
8458
8459                 TAILQ_REMOVE(&p->tables, elem, node);
8460                 free(elem->fields);
8461                 free(elem->actions);
8462                 free(elem->default_action_data);
8463                 free(elem);
8464         }
8465
8466         /* Table types. */
8467         for ( ; ; ) {
8468                 struct table_type *elem;
8469
8470                 elem = TAILQ_FIRST(&p->table_types);
8471                 if (!elem)
8472                         break;
8473
8474                 TAILQ_REMOVE(&p->table_types, elem, node);
8475                 free(elem);
8476         }
8477 }
8478
8479 /*
8480  * Selector.
8481  */
8482 static struct selector *
8483 selector_find(struct rte_swx_pipeline *p, const char *name)
8484 {
8485         struct selector *s;
8486
8487         TAILQ_FOREACH(s, &p->selectors, node)
8488                 if (strcmp(s->name, name) == 0)
8489                         return s;
8490
8491         return NULL;
8492 }
8493
8494 static struct selector *
8495 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8496 {
8497         struct selector *s = NULL;
8498
8499         TAILQ_FOREACH(s, &p->selectors, node)
8500                 if (s->id == id)
8501                         return s;
8502
8503         return NULL;
8504 }
8505
8506 static int
8507 selector_fields_check(struct rte_swx_pipeline *p,
8508                       struct rte_swx_pipeline_selector_params *params,
8509                       struct header **header)
8510 {
8511         struct header *h0 = NULL;
8512         struct field *hf, *mf;
8513         uint32_t i;
8514
8515         /* Return if no selector fields. */
8516         if (!params->n_selector_fields || !params->selector_field_names)
8517                 return -EINVAL;
8518
8519         /* Check that all the selector fields either belong to the same header
8520          * or are all meta-data fields.
8521          */
8522         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8523         mf = metadata_field_parse(p, params->selector_field_names[0]);
8524         if (!hf && !mf)
8525                 return -EINVAL;
8526
8527         for (i = 1; i < params->n_selector_fields; i++)
8528                 if (h0) {
8529                         struct header *h;
8530
8531                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8532                         if (!hf || (h->id != h0->id))
8533                                 return -EINVAL;
8534                 } else {
8535                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8536                         if (!mf)
8537                                 return -EINVAL;
8538                 }
8539
8540         /* Check that there are no duplicated match fields. */
8541         for (i = 0; i < params->n_selector_fields; i++) {
8542                 const char *field_name = params->selector_field_names[i];
8543                 uint32_t j;
8544
8545                 for (j = i + 1; j < params->n_selector_fields; j++)
8546                         if (!strcmp(params->selector_field_names[j], field_name))
8547                                 return -EINVAL;
8548         }
8549
8550         /* Return. */
8551         if (header)
8552                 *header = h0;
8553
8554         return 0;
8555 }
8556
8557 int
8558 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8559                                  const char *name,
8560                                  struct rte_swx_pipeline_selector_params *params)
8561 {
8562         struct selector *s;
8563         struct header *selector_header = NULL;
8564         struct field *group_id_field, *member_id_field;
8565         uint32_t i;
8566         int status = 0;
8567
8568         CHECK(p, EINVAL);
8569
8570         CHECK_NAME(name, EINVAL);
8571         CHECK(!table_find(p, name), EEXIST);
8572         CHECK(!selector_find(p, name), EEXIST);
8573         CHECK(!learner_find(p, name), EEXIST);
8574
8575         CHECK(params, EINVAL);
8576
8577         CHECK_NAME(params->group_id_field_name, EINVAL);
8578         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8579         CHECK(group_id_field, EINVAL);
8580
8581         for (i = 0; i < params->n_selector_fields; i++) {
8582                 const char *field_name = params->selector_field_names[i];
8583
8584                 CHECK_NAME(field_name, EINVAL);
8585         }
8586         status = selector_fields_check(p, params, &selector_header);
8587         if (status)
8588                 return status;
8589
8590         CHECK_NAME(params->member_id_field_name, EINVAL);
8591         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8592         CHECK(member_id_field, EINVAL);
8593
8594         CHECK(params->n_groups_max, EINVAL);
8595
8596         CHECK(params->n_members_per_group_max, EINVAL);
8597
8598         /* Memory allocation. */
8599         s = calloc(1, sizeof(struct selector));
8600         if (!s) {
8601                 status = -ENOMEM;
8602                 goto error;
8603         }
8604
8605         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8606         if (!s->selector_fields) {
8607                 status = -ENOMEM;
8608                 goto error;
8609         }
8610
8611         /* Node initialization. */
8612         strcpy(s->name, name);
8613
8614         s->group_id_field = group_id_field;
8615
8616         for (i = 0; i < params->n_selector_fields; i++) {
8617                 const char *field_name = params->selector_field_names[i];
8618
8619                 s->selector_fields[i] = selector_header ?
8620                         header_field_parse(p, field_name, NULL) :
8621                         metadata_field_parse(p, field_name);
8622         }
8623
8624         s->n_selector_fields = params->n_selector_fields;
8625
8626         s->selector_header = selector_header;
8627
8628         s->member_id_field = member_id_field;
8629
8630         s->n_groups_max = params->n_groups_max;
8631
8632         s->n_members_per_group_max = params->n_members_per_group_max;
8633
8634         s->id = p->n_selectors;
8635
8636         /* Node add to tailq. */
8637         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8638         p->n_selectors++;
8639
8640         return 0;
8641
8642 error:
8643         if (!s)
8644                 return status;
8645
8646         free(s->selector_fields);
8647
8648         free(s);
8649
8650         return status;
8651 }
8652
8653 static void
8654 selector_params_free(struct rte_swx_table_selector_params *params)
8655 {
8656         if (!params)
8657                 return;
8658
8659         free(params->selector_mask);
8660
8661         free(params);
8662 }
8663
8664 static struct rte_swx_table_selector_params *
8665 selector_table_params_get(struct selector *s)
8666 {
8667         struct rte_swx_table_selector_params *params = NULL;
8668         struct field *first, *last;
8669         uint32_t i;
8670
8671         /* Memory allocation. */
8672         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8673         if (!params)
8674                 goto error;
8675
8676         /* Group ID. */
8677         params->group_id_offset = s->group_id_field->offset / 8;
8678
8679         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8680         first = s->selector_fields[0];
8681         last = s->selector_fields[0];
8682
8683         for (i = 0; i < s->n_selector_fields; i++) {
8684                 struct field *f = s->selector_fields[i];
8685
8686                 if (f->offset < first->offset)
8687                         first = f;
8688
8689                 if (f->offset > last->offset)
8690                         last = f;
8691         }
8692
8693         /* Selector offset and size. */
8694         params->selector_offset = first->offset / 8;
8695         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8696
8697         /* Memory allocation. */
8698         params->selector_mask = calloc(1, params->selector_size);
8699         if (!params->selector_mask)
8700                 goto error;
8701
8702         /* Selector mask. */
8703         for (i = 0; i < s->n_selector_fields; i++) {
8704                 struct field *f = s->selector_fields[i];
8705                 uint32_t start = (f->offset - first->offset) / 8;
8706                 size_t size = f->n_bits / 8;
8707
8708                 memset(&params->selector_mask[start], 0xFF, size);
8709         }
8710
8711         /* Member ID. */
8712         params->member_id_offset = s->member_id_field->offset / 8;
8713
8714         /* Maximum number of groups. */
8715         params->n_groups_max = s->n_groups_max;
8716
8717         /* Maximum number of members per group. */
8718         params->n_members_per_group_max = s->n_members_per_group_max;
8719
8720         return params;
8721
8722 error:
8723         selector_params_free(params);
8724         return NULL;
8725 }
8726
8727 static void
8728 selector_build_free(struct rte_swx_pipeline *p)
8729 {
8730         uint32_t i;
8731
8732         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8733                 struct thread *t = &p->threads[i];
8734                 uint32_t j;
8735
8736                 if (!t->selectors)
8737                         continue;
8738
8739                 for (j = 0; j < p->n_selectors; j++) {
8740                         struct selector_runtime *r = &t->selectors[j];
8741
8742                         free(r->mailbox);
8743                 }
8744
8745                 free(t->selectors);
8746                 t->selectors = NULL;
8747         }
8748
8749         free(p->selector_stats);
8750         p->selector_stats = NULL;
8751 }
8752
8753 static int
8754 selector_build(struct rte_swx_pipeline *p)
8755 {
8756         uint32_t i;
8757         int status = 0;
8758
8759         /* Per pipeline: selector statistics. */
8760         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8761         if (!p->selector_stats) {
8762                 status = -ENOMEM;
8763                 goto error;
8764         }
8765
8766         /* Per thread: selector run-time. */
8767         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8768                 struct thread *t = &p->threads[i];
8769                 struct selector *s;
8770
8771                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8772                 if (!t->selectors) {
8773                         status = -ENOMEM;
8774                         goto error;
8775                 }
8776
8777                 TAILQ_FOREACH(s, &p->selectors, node) {
8778                         struct selector_runtime *r = &t->selectors[s->id];
8779                         uint64_t size;
8780
8781                         /* r->mailbox. */
8782                         size = rte_swx_table_selector_mailbox_size_get();
8783                         if (size) {
8784                                 r->mailbox = calloc(1, size);
8785                                 if (!r->mailbox) {
8786                                         status = -ENOMEM;
8787                                         goto error;
8788                                 }
8789                         }
8790
8791                         /* r->group_id_buffer. */
8792                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8793
8794                         /* r->selector_buffer. */
8795                         r->selector_buffer = s->selector_header ?
8796                                 &t->structs[s->selector_header->struct_id] :
8797                                 &t->structs[p->metadata_struct_id];
8798
8799                         /* r->member_id_buffer. */
8800                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8801                 }
8802         }
8803
8804         return 0;
8805
8806 error:
8807         selector_build_free(p);
8808         return status;
8809 }
8810
8811 static void
8812 selector_free(struct rte_swx_pipeline *p)
8813 {
8814         selector_build_free(p);
8815
8816         /* Selector tables. */
8817         for ( ; ; ) {
8818                 struct selector *elem;
8819
8820                 elem = TAILQ_FIRST(&p->selectors);
8821                 if (!elem)
8822                         break;
8823
8824                 TAILQ_REMOVE(&p->selectors, elem, node);
8825                 free(elem->selector_fields);
8826                 free(elem);
8827         }
8828 }
8829
8830 /*
8831  * Learner table.
8832  */
8833 static struct learner *
8834 learner_find(struct rte_swx_pipeline *p, const char *name)
8835 {
8836         struct learner *l;
8837
8838         TAILQ_FOREACH(l, &p->learners, node)
8839                 if (!strcmp(l->name, name))
8840                         return l;
8841
8842         return NULL;
8843 }
8844
8845 static struct learner *
8846 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8847 {
8848         struct learner *l = NULL;
8849
8850         TAILQ_FOREACH(l, &p->learners, node)
8851                 if (l->id == id)
8852                         return l;
8853
8854         return NULL;
8855 }
8856
8857 static int
8858 learner_match_fields_check(struct rte_swx_pipeline *p,
8859                            struct rte_swx_pipeline_learner_params *params,
8860                            struct header **header)
8861 {
8862         struct header *h0 = NULL;
8863         struct field *hf, *mf;
8864         uint32_t i;
8865
8866         /* Return if no match fields. */
8867         if (!params->n_fields || !params->field_names)
8868                 return -EINVAL;
8869
8870         /* Check that all the match fields either belong to the same header
8871          * or are all meta-data fields.
8872          */
8873         hf = header_field_parse(p, params->field_names[0], &h0);
8874         mf = metadata_field_parse(p, params->field_names[0]);
8875         if (!hf && !mf)
8876                 return -EINVAL;
8877
8878         for (i = 1; i < params->n_fields; i++)
8879                 if (h0) {
8880                         struct header *h;
8881
8882                         hf = header_field_parse(p, params->field_names[i], &h);
8883                         if (!hf || (h->id != h0->id))
8884                                 return -EINVAL;
8885                 } else {
8886                         mf = metadata_field_parse(p, params->field_names[i]);
8887                         if (!mf)
8888                                 return -EINVAL;
8889                 }
8890
8891         /* Check that there are no duplicated match fields. */
8892         for (i = 0; i < params->n_fields; i++) {
8893                 const char *field_name = params->field_names[i];
8894                 uint32_t j;
8895
8896                 for (j = i + 1; j < params->n_fields; j++)
8897                         if (!strcmp(params->field_names[j], field_name))
8898                                 return -EINVAL;
8899         }
8900
8901         /* Return. */
8902         if (header)
8903                 *header = h0;
8904
8905         return 0;
8906 }
8907
8908 static int
8909 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8910 {
8911         struct struct_type *mst = p->metadata_st, *ast = a->st;
8912         struct field *mf, *af;
8913         uint32_t mf_pos, i;
8914
8915         if (!ast) {
8916                 if (mf_name)
8917                         return -EINVAL;
8918
8919                 return 0;
8920         }
8921
8922         /* Check that mf_name is the name of a valid meta-data field. */
8923         CHECK_NAME(mf_name, EINVAL);
8924         mf = metadata_field_parse(p, mf_name);
8925         CHECK(mf, EINVAL);
8926
8927         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8928          * all the action arguments.
8929          */
8930         mf_pos = mf - mst->fields;
8931         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8932
8933         /* Check that the size of each of the identified meta-data fields matches exactly the size
8934          * of the corresponding action argument.
8935          */
8936         for (i = 0; i < ast->n_fields; i++) {
8937                 mf = &mst->fields[mf_pos + i];
8938                 af = &ast->fields[i];
8939
8940                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8941         }
8942
8943         return 0;
8944 }
8945
8946 static int
8947 learner_action_learning_check(struct rte_swx_pipeline *p,
8948                               struct action *action,
8949                               const char **action_names,
8950                               uint32_t n_actions)
8951 {
8952         uint32_t i;
8953
8954         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8955          * the action passed as argument to the "learn" instruction) is also enabled for the
8956          * current learner table.
8957          */
8958         for (i = 0; i < action->n_instructions; i++) {
8959                 struct instruction *instr = &action->instructions[i];
8960                 uint32_t found = 0, j;
8961
8962                 if (instr->type != INSTR_LEARNER_LEARN)
8963                         continue;
8964
8965                 for (j = 0; j < n_actions; j++) {
8966                         struct action *a;
8967
8968                         a = action_find(p, action_names[j]);
8969                         if (!a)
8970                                 return -EINVAL;
8971
8972                         if (a->id == instr->learn.action_id)
8973                                 found = 1;
8974                 }
8975
8976                 if (!found)
8977                         return -EINVAL;
8978         }
8979
8980         return 0;
8981 }
8982
8983 int
8984 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8985                               const char *name,
8986                               struct rte_swx_pipeline_learner_params *params,
8987                               uint32_t size,
8988                               uint32_t timeout)
8989 {
8990         struct learner *l = NULL;
8991         struct action *default_action;
8992         struct header *header = NULL;
8993         uint32_t action_data_size_max = 0, i;
8994         int status = 0;
8995
8996         CHECK(p, EINVAL);
8997
8998         CHECK_NAME(name, EINVAL);
8999         CHECK(!table_find(p, name), EEXIST);
9000         CHECK(!selector_find(p, name), EEXIST);
9001         CHECK(!learner_find(p, name), EEXIST);
9002
9003         CHECK(params, EINVAL);
9004
9005         /* Match checks. */
9006         status = learner_match_fields_check(p, params, &header);
9007         if (status)
9008                 return status;
9009
9010         /* Action checks. */
9011         CHECK(params->n_actions, EINVAL);
9012
9013         CHECK(params->action_names, EINVAL);
9014         for (i = 0; i < params->n_actions; i++) {
9015                 const char *action_name = params->action_names[i];
9016                 const char *action_field_name = params->action_field_names[i];
9017                 struct action *a;
9018                 uint32_t action_data_size;
9019
9020                 CHECK_NAME(action_name, EINVAL);
9021
9022                 a = action_find(p, action_name);
9023                 CHECK(a, EINVAL);
9024
9025                 status = learner_action_args_check(p, a, action_field_name);
9026                 if (status)
9027                         return status;
9028
9029                 status = learner_action_learning_check(p,
9030                                                        a,
9031                                                        params->action_names,
9032                                                        params->n_actions);
9033                 if (status)
9034                         return status;
9035
9036                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9037                 if (action_data_size > action_data_size_max)
9038                         action_data_size_max = action_data_size;
9039         }
9040
9041         CHECK_NAME(params->default_action_name, EINVAL);
9042         for (i = 0; i < p->n_actions; i++)
9043                 if (!strcmp(params->action_names[i],
9044                             params->default_action_name))
9045                         break;
9046         CHECK(i < params->n_actions, EINVAL);
9047
9048         default_action = action_find(p, params->default_action_name);
9049         CHECK((default_action->st && params->default_action_data) ||
9050               !params->default_action_data, EINVAL);
9051
9052         /* Any other checks. */
9053         CHECK(size, EINVAL);
9054         CHECK(timeout, EINVAL);
9055
9056         /* Memory allocation. */
9057         l = calloc(1, sizeof(struct learner));
9058         if (!l)
9059                 goto nomem;
9060
9061         l->fields = calloc(params->n_fields, sizeof(struct field *));
9062         if (!l->fields)
9063                 goto nomem;
9064
9065         l->actions = calloc(params->n_actions, sizeof(struct action *));
9066         if (!l->actions)
9067                 goto nomem;
9068
9069         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9070         if (!l->action_arg)
9071                 goto nomem;
9072
9073         if (action_data_size_max) {
9074                 l->default_action_data = calloc(1, action_data_size_max);
9075                 if (!l->default_action_data)
9076                         goto nomem;
9077         }
9078
9079         /* Node initialization. */
9080         strcpy(l->name, name);
9081
9082         for (i = 0; i < params->n_fields; i++) {
9083                 const char *field_name = params->field_names[i];
9084
9085                 l->fields[i] = header ?
9086                         header_field_parse(p, field_name, NULL) :
9087                         metadata_field_parse(p, field_name);
9088         }
9089
9090         l->n_fields = params->n_fields;
9091
9092         l->header = header;
9093
9094         for (i = 0; i < params->n_actions; i++) {
9095                 const char *mf_name = params->action_field_names[i];
9096
9097                 l->actions[i] = action_find(p, params->action_names[i]);
9098
9099                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9100         }
9101
9102         l->default_action = default_action;
9103
9104         if (default_action->st)
9105                 memcpy(l->default_action_data,
9106                        params->default_action_data,
9107                        default_action->st->n_bits / 8);
9108
9109         l->n_actions = params->n_actions;
9110
9111         l->default_action_is_const = params->default_action_is_const;
9112
9113         l->action_data_size_max = action_data_size_max;
9114
9115         l->size = size;
9116
9117         l->timeout = timeout;
9118
9119         l->id = p->n_learners;
9120
9121         /* Node add to tailq. */
9122         TAILQ_INSERT_TAIL(&p->learners, l, node);
9123         p->n_learners++;
9124
9125         return 0;
9126
9127 nomem:
9128         if (!l)
9129                 return -ENOMEM;
9130
9131         free(l->action_arg);
9132         free(l->actions);
9133         free(l->fields);
9134         free(l);
9135
9136         return -ENOMEM;
9137 }
9138
9139 static void
9140 learner_params_free(struct rte_swx_table_learner_params *params)
9141 {
9142         if (!params)
9143                 return;
9144
9145         free(params->key_mask0);
9146
9147         free(params);
9148 }
9149
9150 static struct rte_swx_table_learner_params *
9151 learner_params_get(struct learner *l)
9152 {
9153         struct rte_swx_table_learner_params *params = NULL;
9154         struct field *first, *last;
9155         uint32_t i;
9156
9157         /* Memory allocation. */
9158         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9159         if (!params)
9160                 goto error;
9161
9162         /* Find first (smallest offset) and last (biggest offset) match fields. */
9163         first = l->fields[0];
9164         last = l->fields[0];
9165
9166         for (i = 0; i < l->n_fields; i++) {
9167                 struct field *f = l->fields[i];
9168
9169                 if (f->offset < first->offset)
9170                         first = f;
9171
9172                 if (f->offset > last->offset)
9173                         last = f;
9174         }
9175
9176         /* Key offset and size. */
9177         params->key_offset = first->offset / 8;
9178         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9179
9180         /* Memory allocation. */
9181         params->key_mask0 = calloc(1, params->key_size);
9182         if (!params->key_mask0)
9183                 goto error;
9184
9185         /* Key mask. */
9186         for (i = 0; i < l->n_fields; i++) {
9187                 struct field *f = l->fields[i];
9188                 uint32_t start = (f->offset - first->offset) / 8;
9189                 size_t size = f->n_bits / 8;
9190
9191                 memset(&params->key_mask0[start], 0xFF, size);
9192         }
9193
9194         /* Action data size. */
9195         params->action_data_size = l->action_data_size_max;
9196
9197         /* Maximum number of keys. */
9198         params->n_keys_max = l->size;
9199
9200         /* Timeout. */
9201         params->key_timeout = l->timeout;
9202
9203         return params;
9204
9205 error:
9206         learner_params_free(params);
9207         return NULL;
9208 }
9209
9210 static void
9211 learner_build_free(struct rte_swx_pipeline *p)
9212 {
9213         uint32_t i;
9214
9215         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9216                 struct thread *t = &p->threads[i];
9217                 uint32_t j;
9218
9219                 if (!t->learners)
9220                         continue;
9221
9222                 for (j = 0; j < p->n_learners; j++) {
9223                         struct learner_runtime *r = &t->learners[j];
9224
9225                         free(r->mailbox);
9226                         free(r->action_data);
9227                 }
9228
9229                 free(t->learners);
9230                 t->learners = NULL;
9231         }
9232
9233         if (p->learner_stats) {
9234                 for (i = 0; i < p->n_learners; i++)
9235                         free(p->learner_stats[i].n_pkts_action);
9236
9237                 free(p->learner_stats);
9238         }
9239 }
9240
9241 static int
9242 learner_build(struct rte_swx_pipeline *p)
9243 {
9244         uint32_t i;
9245         int status = 0;
9246
9247         /* Per pipeline: learner statistics. */
9248         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9249         CHECK(p->learner_stats, ENOMEM);
9250
9251         for (i = 0; i < p->n_learners; i++) {
9252                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9253                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9254         }
9255
9256         /* Per thread: learner run-time. */
9257         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9258                 struct thread *t = &p->threads[i];
9259                 struct learner *l;
9260
9261                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9262                 if (!t->learners) {
9263                         status = -ENOMEM;
9264                         goto error;
9265                 }
9266
9267                 TAILQ_FOREACH(l, &p->learners, node) {
9268                         struct learner_runtime *r = &t->learners[l->id];
9269                         uint64_t size;
9270                         uint32_t j;
9271
9272                         /* r->mailbox. */
9273                         size = rte_swx_table_learner_mailbox_size_get();
9274                         if (size) {
9275                                 r->mailbox = calloc(1, size);
9276                                 if (!r->mailbox) {
9277                                         status = -ENOMEM;
9278                                         goto error;
9279                                 }
9280                         }
9281
9282                         /* r->key. */
9283                         r->key = l->header ?
9284                                 &t->structs[l->header->struct_id] :
9285                                 &t->structs[p->metadata_struct_id];
9286
9287                         /* r->action_data. */
9288                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9289                         if (!r->action_data) {
9290                                 status = -ENOMEM;
9291                                 goto error;
9292                         }
9293
9294                         for (j = 0; j < l->n_actions; j++) {
9295                                 struct action *a = l->actions[j];
9296                                 struct field *mf = l->action_arg[j];
9297                                 uint8_t *m = t->structs[p->metadata_struct_id];
9298
9299                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9300                         }
9301                 }
9302         }
9303
9304         return 0;
9305
9306 error:
9307         learner_build_free(p);
9308         return status;
9309 }
9310
9311 static void
9312 learner_free(struct rte_swx_pipeline *p)
9313 {
9314         learner_build_free(p);
9315
9316         /* Learner tables. */
9317         for ( ; ; ) {
9318                 struct learner *l;
9319
9320                 l = TAILQ_FIRST(&p->learners);
9321                 if (!l)
9322                         break;
9323
9324                 TAILQ_REMOVE(&p->learners, l, node);
9325                 free(l->fields);
9326                 free(l->actions);
9327                 free(l->action_arg);
9328                 free(l->default_action_data);
9329                 free(l);
9330         }
9331 }
9332
9333 /*
9334  * Table state.
9335  */
9336 static int
9337 table_state_build(struct rte_swx_pipeline *p)
9338 {
9339         struct table *table;
9340         struct selector *s;
9341         struct learner *l;
9342
9343         p->table_state = calloc(p->n_tables + p->n_selectors,
9344                                 sizeof(struct rte_swx_table_state));
9345         CHECK(p->table_state, ENOMEM);
9346
9347         TAILQ_FOREACH(table, &p->tables, node) {
9348                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9349
9350                 if (table->type) {
9351                         struct rte_swx_table_params *params;
9352
9353                         /* ts->obj. */
9354                         params = table_params_get(table);
9355                         CHECK(params, ENOMEM);
9356
9357                         ts->obj = table->type->ops.create(params,
9358                                 NULL,
9359                                 table->args,
9360                                 p->numa_node);
9361
9362                         table_params_free(params);
9363                         CHECK(ts->obj, ENODEV);
9364                 }
9365
9366                 /* ts->default_action_data. */
9367                 if (table->action_data_size_max) {
9368                         ts->default_action_data =
9369                                 malloc(table->action_data_size_max);
9370                         CHECK(ts->default_action_data, ENOMEM);
9371
9372                         memcpy(ts->default_action_data,
9373                                table->default_action_data,
9374                                table->action_data_size_max);
9375                 }
9376
9377                 /* ts->default_action_id. */
9378                 ts->default_action_id = table->default_action->id;
9379         }
9380
9381         TAILQ_FOREACH(s, &p->selectors, node) {
9382                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9383                 struct rte_swx_table_selector_params *params;
9384
9385                 /* ts->obj. */
9386                 params = selector_table_params_get(s);
9387                 CHECK(params, ENOMEM);
9388
9389                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9390
9391                 selector_params_free(params);
9392                 CHECK(ts->obj, ENODEV);
9393         }
9394
9395         TAILQ_FOREACH(l, &p->learners, node) {
9396                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9397                         p->n_selectors + l->id];
9398                 struct rte_swx_table_learner_params *params;
9399
9400                 /* ts->obj. */
9401                 params = learner_params_get(l);
9402                 CHECK(params, ENOMEM);
9403
9404                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9405                 learner_params_free(params);
9406                 CHECK(ts->obj, ENODEV);
9407
9408                 /* ts->default_action_data. */
9409                 if (l->action_data_size_max) {
9410                         ts->default_action_data = malloc(l->action_data_size_max);
9411                         CHECK(ts->default_action_data, ENOMEM);
9412
9413                         memcpy(ts->default_action_data,
9414                                l->default_action_data,
9415                                l->action_data_size_max);
9416                 }
9417
9418                 /* ts->default_action_id. */
9419                 ts->default_action_id = l->default_action->id;
9420         }
9421
9422         return 0;
9423 }
9424
9425 static void
9426 table_state_build_free(struct rte_swx_pipeline *p)
9427 {
9428         uint32_t i;
9429
9430         if (!p->table_state)
9431                 return;
9432
9433         for (i = 0; i < p->n_tables; i++) {
9434                 struct rte_swx_table_state *ts = &p->table_state[i];
9435                 struct table *table = table_find_by_id(p, i);
9436
9437                 /* ts->obj. */
9438                 if (table->type && ts->obj)
9439                         table->type->ops.free(ts->obj);
9440
9441                 /* ts->default_action_data. */
9442                 free(ts->default_action_data);
9443         }
9444
9445         for (i = 0; i < p->n_selectors; i++) {
9446                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9447
9448                 /* ts->obj. */
9449                 if (ts->obj)
9450                         rte_swx_table_selector_free(ts->obj);
9451         }
9452
9453         for (i = 0; i < p->n_learners; i++) {
9454                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9455
9456                 /* ts->obj. */
9457                 if (ts->obj)
9458                         rte_swx_table_learner_free(ts->obj);
9459
9460                 /* ts->default_action_data. */
9461                 free(ts->default_action_data);
9462         }
9463
9464         free(p->table_state);
9465         p->table_state = NULL;
9466 }
9467
9468 static void
9469 table_state_free(struct rte_swx_pipeline *p)
9470 {
9471         table_state_build_free(p);
9472 }
9473
9474 /*
9475  * Register array.
9476  */
9477 static struct regarray *
9478 regarray_find(struct rte_swx_pipeline *p, const char *name)
9479 {
9480         struct regarray *elem;
9481
9482         TAILQ_FOREACH(elem, &p->regarrays, node)
9483                 if (!strcmp(elem->name, name))
9484                         return elem;
9485
9486         return NULL;
9487 }
9488
9489 static struct regarray *
9490 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9491 {
9492         struct regarray *elem = NULL;
9493
9494         TAILQ_FOREACH(elem, &p->regarrays, node)
9495                 if (elem->id == id)
9496                         return elem;
9497
9498         return NULL;
9499 }
9500
9501 int
9502 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9503                               const char *name,
9504                               uint32_t size,
9505                               uint64_t init_val)
9506 {
9507         struct regarray *r;
9508
9509         CHECK(p, EINVAL);
9510
9511         CHECK_NAME(name, EINVAL);
9512         CHECK(!regarray_find(p, name), EEXIST);
9513
9514         CHECK(size, EINVAL);
9515         size = rte_align32pow2(size);
9516
9517         /* Memory allocation. */
9518         r = calloc(1, sizeof(struct regarray));
9519         CHECK(r, ENOMEM);
9520
9521         /* Node initialization. */
9522         strcpy(r->name, name);
9523         r->init_val = init_val;
9524         r->size = size;
9525         r->id = p->n_regarrays;
9526
9527         /* Node add to tailq. */
9528         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9529         p->n_regarrays++;
9530
9531         return 0;
9532 }
9533
9534 static int
9535 regarray_build(struct rte_swx_pipeline *p)
9536 {
9537         struct regarray *regarray;
9538
9539         if (!p->n_regarrays)
9540                 return 0;
9541
9542         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9543         CHECK(p->regarray_runtime, ENOMEM);
9544
9545         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9546                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9547                 uint32_t i;
9548
9549                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9550                                          RTE_CACHE_LINE_SIZE,
9551                                          p->numa_node);
9552                 CHECK(r->regarray, ENOMEM);
9553
9554                 if (regarray->init_val)
9555                         for (i = 0; i < regarray->size; i++)
9556                                 r->regarray[i] = regarray->init_val;
9557
9558                 r->size_mask = regarray->size - 1;
9559         }
9560
9561         return 0;
9562 }
9563
9564 static void
9565 regarray_build_free(struct rte_swx_pipeline *p)
9566 {
9567         uint32_t i;
9568
9569         if (!p->regarray_runtime)
9570                 return;
9571
9572         for (i = 0; i < p->n_regarrays; i++) {
9573                 struct regarray *regarray = regarray_find_by_id(p, i);
9574                 struct regarray_runtime *r = &p->regarray_runtime[i];
9575
9576                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9577         }
9578
9579         free(p->regarray_runtime);
9580         p->regarray_runtime = NULL;
9581 }
9582
9583 static void
9584 regarray_free(struct rte_swx_pipeline *p)
9585 {
9586         regarray_build_free(p);
9587
9588         for ( ; ; ) {
9589                 struct regarray *elem;
9590
9591                 elem = TAILQ_FIRST(&p->regarrays);
9592                 if (!elem)
9593                         break;
9594
9595                 TAILQ_REMOVE(&p->regarrays, elem, node);
9596                 free(elem);
9597         }
9598 }
9599
9600 /*
9601  * Meter array.
9602  */
9603 static struct meter_profile *
9604 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9605 {
9606         struct meter_profile *elem;
9607
9608         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9609                 if (!strcmp(elem->name, name))
9610                         return elem;
9611
9612         return NULL;
9613 }
9614
9615 static struct metarray *
9616 metarray_find(struct rte_swx_pipeline *p, const char *name)
9617 {
9618         struct metarray *elem;
9619
9620         TAILQ_FOREACH(elem, &p->metarrays, node)
9621                 if (!strcmp(elem->name, name))
9622                         return elem;
9623
9624         return NULL;
9625 }
9626
9627 static struct metarray *
9628 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9629 {
9630         struct metarray *elem = NULL;
9631
9632         TAILQ_FOREACH(elem, &p->metarrays, node)
9633                 if (elem->id == id)
9634                         return elem;
9635
9636         return NULL;
9637 }
9638
9639 int
9640 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9641                                  const char *name,
9642                                  uint32_t size)
9643 {
9644         struct metarray *m;
9645
9646         CHECK(p, EINVAL);
9647
9648         CHECK_NAME(name, EINVAL);
9649         CHECK(!metarray_find(p, name), EEXIST);
9650
9651         CHECK(size, EINVAL);
9652         size = rte_align32pow2(size);
9653
9654         /* Memory allocation. */
9655         m = calloc(1, sizeof(struct metarray));
9656         CHECK(m, ENOMEM);
9657
9658         /* Node initialization. */
9659         strcpy(m->name, name);
9660         m->size = size;
9661         m->id = p->n_metarrays;
9662
9663         /* Node add to tailq. */
9664         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9665         p->n_metarrays++;
9666
9667         return 0;
9668 }
9669
9670 struct meter_profile meter_profile_default = {
9671         .node = {0},
9672         .name = "",
9673         .params = {0},
9674
9675         .profile = {
9676                 .cbs = 10000,
9677                 .pbs = 10000,
9678                 .cir_period = 1,
9679                 .cir_bytes_per_period = 1,
9680                 .pir_period = 1,
9681                 .pir_bytes_per_period = 1,
9682         },
9683
9684         .n_users = 0,
9685 };
9686
9687 static void
9688 meter_init(struct meter *m)
9689 {
9690         memset(m, 0, sizeof(struct meter));
9691         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9692         m->profile = &meter_profile_default;
9693         m->color_mask = RTE_COLOR_GREEN;
9694
9695         meter_profile_default.n_users++;
9696 }
9697
9698 static int
9699 metarray_build(struct rte_swx_pipeline *p)
9700 {
9701         struct metarray *m;
9702
9703         if (!p->n_metarrays)
9704                 return 0;
9705
9706         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9707         CHECK(p->metarray_runtime, ENOMEM);
9708
9709         TAILQ_FOREACH(m, &p->metarrays, node) {
9710                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9711                 uint32_t i;
9712
9713                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9714                                          RTE_CACHE_LINE_SIZE,
9715                                          p->numa_node);
9716                 CHECK(r->metarray, ENOMEM);
9717
9718                 for (i = 0; i < m->size; i++)
9719                         meter_init(&r->metarray[i]);
9720
9721                 r->size_mask = m->size - 1;
9722         }
9723
9724         return 0;
9725 }
9726
9727 static void
9728 metarray_build_free(struct rte_swx_pipeline *p)
9729 {
9730         uint32_t i;
9731
9732         if (!p->metarray_runtime)
9733                 return;
9734
9735         for (i = 0; i < p->n_metarrays; i++) {
9736                 struct metarray *m = metarray_find_by_id(p, i);
9737                 struct metarray_runtime *r = &p->metarray_runtime[i];
9738
9739                 env_free(r->metarray, m->size * sizeof(struct meter));
9740         }
9741
9742         free(p->metarray_runtime);
9743         p->metarray_runtime = NULL;
9744 }
9745
9746 static void
9747 metarray_free(struct rte_swx_pipeline *p)
9748 {
9749         metarray_build_free(p);
9750
9751         /* Meter arrays. */
9752         for ( ; ; ) {
9753                 struct metarray *elem;
9754
9755                 elem = TAILQ_FIRST(&p->metarrays);
9756                 if (!elem)
9757                         break;
9758
9759                 TAILQ_REMOVE(&p->metarrays, elem, node);
9760                 free(elem);
9761         }
9762
9763         /* Meter profiles. */
9764         for ( ; ; ) {
9765                 struct meter_profile *elem;
9766
9767                 elem = TAILQ_FIRST(&p->meter_profiles);
9768                 if (!elem)
9769                         break;
9770
9771                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9772                 free(elem);
9773         }
9774 }
9775
9776 /*
9777  * Pipeline.
9778  */
9779 int
9780 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9781 {
9782         struct rte_swx_pipeline *pipeline;
9783
9784         /* Check input parameters. */
9785         CHECK(p, EINVAL);
9786
9787         /* Memory allocation. */
9788         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9789         CHECK(pipeline, ENOMEM);
9790
9791         /* Initialization. */
9792         TAILQ_INIT(&pipeline->struct_types);
9793         TAILQ_INIT(&pipeline->port_in_types);
9794         TAILQ_INIT(&pipeline->ports_in);
9795         TAILQ_INIT(&pipeline->port_out_types);
9796         TAILQ_INIT(&pipeline->ports_out);
9797         TAILQ_INIT(&pipeline->extern_types);
9798         TAILQ_INIT(&pipeline->extern_objs);
9799         TAILQ_INIT(&pipeline->extern_funcs);
9800         TAILQ_INIT(&pipeline->headers);
9801         TAILQ_INIT(&pipeline->actions);
9802         TAILQ_INIT(&pipeline->table_types);
9803         TAILQ_INIT(&pipeline->tables);
9804         TAILQ_INIT(&pipeline->selectors);
9805         TAILQ_INIT(&pipeline->learners);
9806         TAILQ_INIT(&pipeline->regarrays);
9807         TAILQ_INIT(&pipeline->meter_profiles);
9808         TAILQ_INIT(&pipeline->metarrays);
9809
9810         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9811         pipeline->numa_node = numa_node;
9812
9813         *p = pipeline;
9814         return 0;
9815 }
9816
9817 void
9818 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9819 {
9820         if (!p)
9821                 return;
9822
9823         free(p->instructions);
9824
9825         metarray_free(p);
9826         regarray_free(p);
9827         table_state_free(p);
9828         learner_free(p);
9829         selector_free(p);
9830         table_free(p);
9831         action_free(p);
9832         metadata_free(p);
9833         header_free(p);
9834         extern_func_free(p);
9835         extern_obj_free(p);
9836         port_out_free(p);
9837         port_in_free(p);
9838         struct_free(p);
9839
9840         free(p);
9841 }
9842
9843 int
9844 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9845                                      const char **instructions,
9846                                      uint32_t n_instructions)
9847 {
9848         int err;
9849         uint32_t i;
9850
9851         err = instruction_config(p, NULL, instructions, n_instructions);
9852         if (err)
9853                 return err;
9854
9855         /* Thread instruction pointer reset. */
9856         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9857                 struct thread *t = &p->threads[i];
9858
9859                 thread_ip_reset(p, t);
9860         }
9861
9862         return 0;
9863 }
9864
9865 int
9866 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9867 {
9868         int status;
9869
9870         CHECK(p, EINVAL);
9871         CHECK(p->build_done == 0, EEXIST);
9872
9873         status = port_in_build(p);
9874         if (status)
9875                 goto error;
9876
9877         status = port_out_build(p);
9878         if (status)
9879                 goto error;
9880
9881         status = struct_build(p);
9882         if (status)
9883                 goto error;
9884
9885         status = extern_obj_build(p);
9886         if (status)
9887                 goto error;
9888
9889         status = extern_func_build(p);
9890         if (status)
9891                 goto error;
9892
9893         status = header_build(p);
9894         if (status)
9895                 goto error;
9896
9897         status = metadata_build(p);
9898         if (status)
9899                 goto error;
9900
9901         status = action_build(p);
9902         if (status)
9903                 goto error;
9904
9905         status = table_build(p);
9906         if (status)
9907                 goto error;
9908
9909         status = selector_build(p);
9910         if (status)
9911                 goto error;
9912
9913         status = learner_build(p);
9914         if (status)
9915                 goto error;
9916
9917         status = table_state_build(p);
9918         if (status)
9919                 goto error;
9920
9921         status = regarray_build(p);
9922         if (status)
9923                 goto error;
9924
9925         status = metarray_build(p);
9926         if (status)
9927                 goto error;
9928
9929         p->build_done = 1;
9930         return 0;
9931
9932 error:
9933         metarray_build_free(p);
9934         regarray_build_free(p);
9935         table_state_build_free(p);
9936         learner_build_free(p);
9937         selector_build_free(p);
9938         table_build_free(p);
9939         action_build_free(p);
9940         metadata_build_free(p);
9941         header_build_free(p);
9942         extern_func_build_free(p);
9943         extern_obj_build_free(p);
9944         port_out_build_free(p);
9945         port_in_build_free(p);
9946         struct_build_free(p);
9947
9948         return status;
9949 }
9950
9951 void
9952 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9953 {
9954         uint32_t i;
9955
9956         for (i = 0; i < n_instructions; i++)
9957                 instr_exec(p);
9958 }
9959
9960 void
9961 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9962 {
9963         uint32_t i;
9964
9965         for (i = 0; i < p->n_ports_out; i++) {
9966                 struct port_out_runtime *port = &p->out[i];
9967
9968                 if (port->flush)
9969                         port->flush(port->obj);
9970         }
9971 }
9972
9973 /*
9974  * Control.
9975  */
9976 int
9977 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9978                               struct rte_swx_ctl_pipeline_info *pipeline)
9979 {
9980         struct action *action;
9981         struct table *table;
9982         uint32_t n_actions = 0, n_tables = 0;
9983
9984         if (!p || !pipeline)
9985                 return -EINVAL;
9986
9987         TAILQ_FOREACH(action, &p->actions, node)
9988                 n_actions++;
9989
9990         TAILQ_FOREACH(table, &p->tables, node)
9991                 n_tables++;
9992
9993         pipeline->n_ports_in = p->n_ports_in;
9994         pipeline->n_ports_out = p->n_ports_out;
9995         pipeline->n_actions = n_actions;
9996         pipeline->n_tables = n_tables;
9997         pipeline->n_selectors = p->n_selectors;
9998         pipeline->n_learners = p->n_learners;
9999         pipeline->n_regarrays = p->n_regarrays;
10000         pipeline->n_metarrays = p->n_metarrays;
10001
10002         return 0;
10003 }
10004
10005 int
10006 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10007 {
10008         if (!p || !numa_node)
10009                 return -EINVAL;
10010
10011         *numa_node = p->numa_node;
10012         return 0;
10013 }
10014
10015 int
10016 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10017                             uint32_t action_id,
10018                             struct rte_swx_ctl_action_info *action)
10019 {
10020         struct action *a = NULL;
10021
10022         if (!p || (action_id >= p->n_actions) || !action)
10023                 return -EINVAL;
10024
10025         a = action_find_by_id(p, action_id);
10026         if (!a)
10027                 return -EINVAL;
10028
10029         strcpy(action->name, a->name);
10030         action->n_args = a->st ? a->st->n_fields : 0;
10031         return 0;
10032 }
10033
10034 int
10035 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10036                                 uint32_t action_id,
10037                                 uint32_t action_arg_id,
10038                                 struct rte_swx_ctl_action_arg_info *action_arg)
10039 {
10040         struct action *a = NULL;
10041         struct field *arg = NULL;
10042
10043         if (!p || (action_id >= p->n_actions) || !action_arg)
10044                 return -EINVAL;
10045
10046         a = action_find_by_id(p, action_id);
10047         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10048                 return -EINVAL;
10049
10050         arg = &a->st->fields[action_arg_id];
10051         strcpy(action_arg->name, arg->name);
10052         action_arg->n_bits = arg->n_bits;
10053         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10054
10055         return 0;
10056 }
10057
10058 int
10059 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10060                            uint32_t table_id,
10061                            struct rte_swx_ctl_table_info *table)
10062 {
10063         struct table *t = NULL;
10064
10065         if (!p || !table)
10066                 return -EINVAL;
10067
10068         t = table_find_by_id(p, table_id);
10069         if (!t)
10070                 return -EINVAL;
10071
10072         strcpy(table->name, t->name);
10073         strcpy(table->args, t->args);
10074         table->n_match_fields = t->n_fields;
10075         table->n_actions = t->n_actions;
10076         table->default_action_is_const = t->default_action_is_const;
10077         table->size = t->size;
10078         return 0;
10079 }
10080
10081 int
10082 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10083         uint32_t table_id,
10084         uint32_t match_field_id,
10085         struct rte_swx_ctl_table_match_field_info *match_field)
10086 {
10087         struct table *t;
10088         struct match_field *f;
10089
10090         if (!p || (table_id >= p->n_tables) || !match_field)
10091                 return -EINVAL;
10092
10093         t = table_find_by_id(p, table_id);
10094         if (!t || (match_field_id >= t->n_fields))
10095                 return -EINVAL;
10096
10097         f = &t->fields[match_field_id];
10098         match_field->match_type = f->match_type;
10099         match_field->is_header = t->header ? 1 : 0;
10100         match_field->n_bits = f->field->n_bits;
10101         match_field->offset = f->field->offset;
10102
10103         return 0;
10104 }
10105
10106 int
10107 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10108         uint32_t table_id,
10109         uint32_t table_action_id,
10110         struct rte_swx_ctl_table_action_info *table_action)
10111 {
10112         struct table *t;
10113
10114         if (!p || (table_id >= p->n_tables) || !table_action)
10115                 return -EINVAL;
10116
10117         t = table_find_by_id(p, table_id);
10118         if (!t || (table_action_id >= t->n_actions))
10119                 return -EINVAL;
10120
10121         table_action->action_id = t->actions[table_action_id]->id;
10122
10123         return 0;
10124 }
10125
10126 int
10127 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10128                           uint32_t table_id,
10129                           struct rte_swx_table_ops *table_ops,
10130                           int *is_stub)
10131 {
10132         struct table *t;
10133
10134         if (!p || (table_id >= p->n_tables))
10135                 return -EINVAL;
10136
10137         t = table_find_by_id(p, table_id);
10138         if (!t)
10139                 return -EINVAL;
10140
10141         if (t->type) {
10142                 if (table_ops)
10143                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10144                 *is_stub = 0;
10145         } else {
10146                 *is_stub = 1;
10147         }
10148
10149         return 0;
10150 }
10151
10152 int
10153 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10154                               uint32_t selector_id,
10155                               struct rte_swx_ctl_selector_info *selector)
10156 {
10157         struct selector *s = NULL;
10158
10159         if (!p || !selector)
10160                 return -EINVAL;
10161
10162         s = selector_find_by_id(p, selector_id);
10163         if (!s)
10164                 return -EINVAL;
10165
10166         strcpy(selector->name, s->name);
10167
10168         selector->n_selector_fields = s->n_selector_fields;
10169         selector->n_groups_max = s->n_groups_max;
10170         selector->n_members_per_group_max = s->n_members_per_group_max;
10171
10172         return 0;
10173 }
10174
10175 int
10176 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10177          uint32_t selector_id,
10178          struct rte_swx_ctl_table_match_field_info *field)
10179 {
10180         struct selector *s;
10181
10182         if (!p || (selector_id >= p->n_selectors) || !field)
10183                 return -EINVAL;
10184
10185         s = selector_find_by_id(p, selector_id);
10186         if (!s)
10187                 return -EINVAL;
10188
10189         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10190         field->is_header = 0;
10191         field->n_bits = s->group_id_field->n_bits;
10192         field->offset = s->group_id_field->offset;
10193
10194         return 0;
10195 }
10196
10197 int
10198 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10199          uint32_t selector_id,
10200          uint32_t selector_field_id,
10201          struct rte_swx_ctl_table_match_field_info *field)
10202 {
10203         struct selector *s;
10204         struct field *f;
10205
10206         if (!p || (selector_id >= p->n_selectors) || !field)
10207                 return -EINVAL;
10208
10209         s = selector_find_by_id(p, selector_id);
10210         if (!s || (selector_field_id >= s->n_selector_fields))
10211                 return -EINVAL;
10212
10213         f = s->selector_fields[selector_field_id];
10214         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10215         field->is_header = s->selector_header ? 1 : 0;
10216         field->n_bits = f->n_bits;
10217         field->offset = f->offset;
10218
10219         return 0;
10220 }
10221
10222 int
10223 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10224          uint32_t selector_id,
10225          struct rte_swx_ctl_table_match_field_info *field)
10226 {
10227         struct selector *s;
10228
10229         if (!p || (selector_id >= p->n_selectors) || !field)
10230                 return -EINVAL;
10231
10232         s = selector_find_by_id(p, selector_id);
10233         if (!s)
10234                 return -EINVAL;
10235
10236         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10237         field->is_header = 0;
10238         field->n_bits = s->member_id_field->n_bits;
10239         field->offset = s->member_id_field->offset;
10240
10241         return 0;
10242 }
10243
10244 int
10245 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10246                              uint32_t learner_id,
10247                              struct rte_swx_ctl_learner_info *learner)
10248 {
10249         struct learner *l = NULL;
10250
10251         if (!p || !learner)
10252                 return -EINVAL;
10253
10254         l = learner_find_by_id(p, learner_id);
10255         if (!l)
10256                 return -EINVAL;
10257
10258         strcpy(learner->name, l->name);
10259
10260         learner->n_match_fields = l->n_fields;
10261         learner->n_actions = l->n_actions;
10262         learner->default_action_is_const = l->default_action_is_const;
10263         learner->size = l->size;
10264
10265         return 0;
10266 }
10267
10268 int
10269 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10270                                          uint32_t learner_id,
10271                                          uint32_t match_field_id,
10272                                          struct rte_swx_ctl_table_match_field_info *match_field)
10273 {
10274         struct learner *l;
10275         struct field *f;
10276
10277         if (!p || (learner_id >= p->n_learners) || !match_field)
10278                 return -EINVAL;
10279
10280         l = learner_find_by_id(p, learner_id);
10281         if (!l || (match_field_id >= l->n_fields))
10282                 return -EINVAL;
10283
10284         f = l->fields[match_field_id];
10285         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10286         match_field->is_header = l->header ? 1 : 0;
10287         match_field->n_bits = f->n_bits;
10288         match_field->offset = f->offset;
10289
10290         return 0;
10291 }
10292
10293 int
10294 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10295                                     uint32_t learner_id,
10296                                     uint32_t learner_action_id,
10297                                     struct rte_swx_ctl_table_action_info *learner_action)
10298 {
10299         struct learner *l;
10300
10301         if (!p || (learner_id >= p->n_learners) || !learner_action)
10302                 return -EINVAL;
10303
10304         l = learner_find_by_id(p, learner_id);
10305         if (!l || (learner_action_id >= l->n_actions))
10306                 return -EINVAL;
10307
10308         learner_action->action_id = l->actions[learner_action_id]->id;
10309
10310         return 0;
10311 }
10312
10313 int
10314 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10315                                  struct rte_swx_table_state **table_state)
10316 {
10317         if (!p || !table_state || !p->build_done)
10318                 return -EINVAL;
10319
10320         *table_state = p->table_state;
10321         return 0;
10322 }
10323
10324 int
10325 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10326                                  struct rte_swx_table_state *table_state)
10327 {
10328         if (!p || !table_state || !p->build_done)
10329                 return -EINVAL;
10330
10331         p->table_state = table_state;
10332         return 0;
10333 }
10334
10335 int
10336 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10337                                         uint32_t port_id,
10338                                         struct rte_swx_port_in_stats *stats)
10339 {
10340         struct port_in *port;
10341
10342         if (!p || !stats)
10343                 return -EINVAL;
10344
10345         port = port_in_find(p, port_id);
10346         if (!port)
10347                 return -EINVAL;
10348
10349         port->type->ops.stats_read(port->obj, stats);
10350         return 0;
10351 }
10352
10353 int
10354 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10355                                          uint32_t port_id,
10356                                          struct rte_swx_port_out_stats *stats)
10357 {
10358         struct port_out *port;
10359
10360         if (!p || !stats)
10361                 return -EINVAL;
10362
10363         port = port_out_find(p, port_id);
10364         if (!port)
10365                 return -EINVAL;
10366
10367         port->type->ops.stats_read(port->obj, stats);
10368         return 0;
10369 }
10370
10371 int
10372 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10373                                       const char *table_name,
10374                                       struct rte_swx_table_stats *stats)
10375 {
10376         struct table *table;
10377         struct table_statistics *table_stats;
10378
10379         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10380                 return -EINVAL;
10381
10382         table = table_find(p, table_name);
10383         if (!table)
10384                 return -EINVAL;
10385
10386         table_stats = &p->table_stats[table->id];
10387
10388         memcpy(stats->n_pkts_action,
10389                table_stats->n_pkts_action,
10390                p->n_actions * sizeof(uint64_t));
10391
10392         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10393         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10394
10395         return 0;
10396 }
10397
10398 int
10399 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10400         const char *selector_name,
10401         struct rte_swx_pipeline_selector_stats *stats)
10402 {
10403         struct selector *s;
10404
10405         if (!p || !selector_name || !selector_name[0] || !stats)
10406                 return -EINVAL;
10407
10408         s = selector_find(p, selector_name);
10409         if (!s)
10410                 return -EINVAL;
10411
10412         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10413
10414         return 0;
10415 }
10416
10417 int
10418 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10419                                         const char *learner_name,
10420                                         struct rte_swx_learner_stats *stats)
10421 {
10422         struct learner *l;
10423         struct learner_statistics *learner_stats;
10424
10425         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10426                 return -EINVAL;
10427
10428         l = learner_find(p, learner_name);
10429         if (!l)
10430                 return -EINVAL;
10431
10432         learner_stats = &p->learner_stats[l->id];
10433
10434         memcpy(stats->n_pkts_action,
10435                learner_stats->n_pkts_action,
10436                p->n_actions * sizeof(uint64_t));
10437
10438         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10439         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10440
10441         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10442         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10443
10444         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10445
10446         return 0;
10447 }
10448
10449 int
10450 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10451                               uint32_t regarray_id,
10452                               struct rte_swx_ctl_regarray_info *regarray)
10453 {
10454         struct regarray *r;
10455
10456         if (!p || !regarray)
10457                 return -EINVAL;
10458
10459         r = regarray_find_by_id(p, regarray_id);
10460         if (!r)
10461                 return -EINVAL;
10462
10463         strcpy(regarray->name, r->name);
10464         regarray->size = r->size;
10465         return 0;
10466 }
10467
10468 int
10469 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10470                                    const char *regarray_name,
10471                                    uint32_t regarray_index,
10472                                    uint64_t *value)
10473 {
10474         struct regarray *regarray;
10475         struct regarray_runtime *r;
10476
10477         if (!p || !regarray_name || !value)
10478                 return -EINVAL;
10479
10480         regarray = regarray_find(p, regarray_name);
10481         if (!regarray || (regarray_index >= regarray->size))
10482                 return -EINVAL;
10483
10484         r = &p->regarray_runtime[regarray->id];
10485         *value = r->regarray[regarray_index];
10486         return 0;
10487 }
10488
10489 int
10490 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10491                                    const char *regarray_name,
10492                                    uint32_t regarray_index,
10493                                    uint64_t value)
10494 {
10495         struct regarray *regarray;
10496         struct regarray_runtime *r;
10497
10498         if (!p || !regarray_name)
10499                 return -EINVAL;
10500
10501         regarray = regarray_find(p, regarray_name);
10502         if (!regarray || (regarray_index >= regarray->size))
10503                 return -EINVAL;
10504
10505         r = &p->regarray_runtime[regarray->id];
10506         r->regarray[regarray_index] = value;
10507         return 0;
10508 }
10509
10510 int
10511 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10512                               uint32_t metarray_id,
10513                               struct rte_swx_ctl_metarray_info *metarray)
10514 {
10515         struct metarray *m;
10516
10517         if (!p || !metarray)
10518                 return -EINVAL;
10519
10520         m = metarray_find_by_id(p, metarray_id);
10521         if (!m)
10522                 return -EINVAL;
10523
10524         strcpy(metarray->name, m->name);
10525         metarray->size = m->size;
10526         return 0;
10527 }
10528
10529 int
10530 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10531                               const char *name,
10532                               struct rte_meter_trtcm_params *params)
10533 {
10534         struct meter_profile *mp;
10535         int status;
10536
10537         CHECK(p, EINVAL);
10538         CHECK_NAME(name, EINVAL);
10539         CHECK(params, EINVAL);
10540         CHECK(!meter_profile_find(p, name), EEXIST);
10541
10542         /* Node allocation. */
10543         mp = calloc(1, sizeof(struct meter_profile));
10544         CHECK(mp, ENOMEM);
10545
10546         /* Node initialization. */
10547         strcpy(mp->name, name);
10548         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10549         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10550         if (status) {
10551                 free(mp);
10552                 CHECK(0, EINVAL);
10553         }
10554
10555         /* Node add to tailq. */
10556         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10557
10558         return 0;
10559 }
10560
10561 int
10562 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10563                                  const char *name)
10564 {
10565         struct meter_profile *mp;
10566
10567         CHECK(p, EINVAL);
10568         CHECK_NAME(name, EINVAL);
10569
10570         mp = meter_profile_find(p, name);
10571         CHECK(mp, EINVAL);
10572         CHECK(!mp->n_users, EBUSY);
10573
10574         /* Remove node from tailq. */
10575         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10576         free(mp);
10577
10578         return 0;
10579 }
10580
10581 int
10582 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10583                         const char *metarray_name,
10584                         uint32_t metarray_index)
10585 {
10586         struct meter_profile *mp_old;
10587         struct metarray *metarray;
10588         struct metarray_runtime *metarray_runtime;
10589         struct meter *m;
10590
10591         CHECK(p, EINVAL);
10592         CHECK_NAME(metarray_name, EINVAL);
10593
10594         metarray = metarray_find(p, metarray_name);
10595         CHECK(metarray, EINVAL);
10596         CHECK(metarray_index < metarray->size, EINVAL);
10597
10598         metarray_runtime = &p->metarray_runtime[metarray->id];
10599         m = &metarray_runtime->metarray[metarray_index];
10600         mp_old = m->profile;
10601
10602         meter_init(m);
10603
10604         mp_old->n_users--;
10605
10606         return 0;
10607 }
10608
10609 int
10610 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10611                       const char *metarray_name,
10612                       uint32_t metarray_index,
10613                       const char *profile_name)
10614 {
10615         struct meter_profile *mp, *mp_old;
10616         struct metarray *metarray;
10617         struct metarray_runtime *metarray_runtime;
10618         struct meter *m;
10619
10620         CHECK(p, EINVAL);
10621         CHECK_NAME(metarray_name, EINVAL);
10622
10623         metarray = metarray_find(p, metarray_name);
10624         CHECK(metarray, EINVAL);
10625         CHECK(metarray_index < metarray->size, EINVAL);
10626
10627         mp = meter_profile_find(p, profile_name);
10628         CHECK(mp, EINVAL);
10629
10630         metarray_runtime = &p->metarray_runtime[metarray->id];
10631         m = &metarray_runtime->metarray[metarray_index];
10632         mp_old = m->profile;
10633
10634         memset(m, 0, sizeof(struct meter));
10635         rte_meter_trtcm_config(&m->m, &mp->profile);
10636         m->profile = mp;
10637         m->color_mask = RTE_COLORS;
10638
10639         mp->n_users++;
10640         mp_old->n_users--;
10641
10642         return 0;
10643 }
10644
10645 int
10646 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10647                              const char *metarray_name,
10648                              uint32_t metarray_index,
10649                              struct rte_swx_ctl_meter_stats *stats)
10650 {
10651         struct metarray *metarray;
10652         struct metarray_runtime *metarray_runtime;
10653         struct meter *m;
10654
10655         CHECK(p, EINVAL);
10656         CHECK_NAME(metarray_name, EINVAL);
10657
10658         metarray = metarray_find(p, metarray_name);
10659         CHECK(metarray, EINVAL);
10660         CHECK(metarray_index < metarray->size, EINVAL);
10661
10662         CHECK(stats, EINVAL);
10663
10664         metarray_runtime = &p->metarray_runtime[metarray->id];
10665         m = &metarray_runtime->metarray[metarray_index];
10666
10667         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10668         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10669
10670         return 0;
10671 }