lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956         uint32_t header_id = ip->valid.header_id;
1957
1958         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
1959
1960         /* Headers. */
1961         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
1962
1963         /* Thread. */
1964         thread_ip_inc(p);
1965 }
1966
1967 /*
1968  * invalidate.
1969  */
1970 static int
1971 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1972                                struct action *action __rte_unused,
1973                                char **tokens,
1974                                int n_tokens,
1975                                struct instruction *instr,
1976                                struct instruction_data *data __rte_unused)
1977 {
1978         struct header *h;
1979
1980         CHECK(n_tokens == 2, EINVAL);
1981
1982         h = header_parse(p, tokens[1]);
1983         CHECK(h, EINVAL);
1984
1985         instr->type = INSTR_HDR_INVALIDATE;
1986         instr->valid.header_id = h->id;
1987         return 0;
1988 }
1989
1990 static inline void
1991 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995         uint32_t header_id = ip->valid.header_id;
1996
1997         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
1998
1999         /* Headers. */
2000         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2001
2002         /* Thread. */
2003         thread_ip_inc(p);
2004 }
2005
2006 /*
2007  * table.
2008  */
2009 static struct table *
2010 table_find(struct rte_swx_pipeline *p, const char *name);
2011
2012 static struct selector *
2013 selector_find(struct rte_swx_pipeline *p, const char *name);
2014
2015 static struct learner *
2016 learner_find(struct rte_swx_pipeline *p, const char *name);
2017
2018 static int
2019 instr_table_translate(struct rte_swx_pipeline *p,
2020                       struct action *action,
2021                       char **tokens,
2022                       int n_tokens,
2023                       struct instruction *instr,
2024                       struct instruction_data *data __rte_unused)
2025 {
2026         struct table *t;
2027         struct selector *s;
2028         struct learner *l;
2029
2030         CHECK(!action, EINVAL);
2031         CHECK(n_tokens == 2, EINVAL);
2032
2033         t = table_find(p, tokens[1]);
2034         if (t) {
2035                 instr->type = INSTR_TABLE;
2036                 instr->table.table_id = t->id;
2037                 return 0;
2038         }
2039
2040         s = selector_find(p, tokens[1]);
2041         if (s) {
2042                 instr->type = INSTR_SELECTOR;
2043                 instr->table.table_id = s->id;
2044                 return 0;
2045         }
2046
2047         l = learner_find(p, tokens[1]);
2048         if (l) {
2049                 instr->type = INSTR_LEARNER;
2050                 instr->table.table_id = l->id;
2051                 return 0;
2052         }
2053
2054         CHECK(0, EINVAL);
2055 }
2056
2057 static inline void
2058 instr_table_exec(struct rte_swx_pipeline *p)
2059 {
2060         struct thread *t = &p->threads[p->thread_id];
2061         struct instruction *ip = t->ip;
2062         uint32_t table_id = ip->table.table_id;
2063         struct rte_swx_table_state *ts = &t->table_state[table_id];
2064         struct table_runtime *table = &t->tables[table_id];
2065         struct table_statistics *stats = &p->table_stats[table_id];
2066         uint64_t action_id, n_pkts_hit, n_pkts_action;
2067         uint8_t *action_data;
2068         int done, hit;
2069
2070         /* Table. */
2071         done = table->func(ts->obj,
2072                            table->mailbox,
2073                            table->key,
2074                            &action_id,
2075                            &action_data,
2076                            &hit);
2077         if (!done) {
2078                 /* Thread. */
2079                 TRACE("[Thread %2u] table %u (not finalized)\n",
2080                       p->thread_id,
2081                       table_id);
2082
2083                 thread_yield(p);
2084                 return;
2085         }
2086
2087         action_id = hit ? action_id : ts->default_action_id;
2088         action_data = hit ? action_data : ts->default_action_data;
2089         n_pkts_hit = stats->n_pkts_hit[hit];
2090         n_pkts_action = stats->n_pkts_action[action_id];
2091
2092         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2093               p->thread_id,
2094               table_id,
2095               hit ? "hit" : "miss",
2096               (uint32_t)action_id);
2097
2098         t->action_id = action_id;
2099         t->structs[0] = action_data;
2100         t->hit = hit;
2101         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2102         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2103
2104         /* Thread. */
2105         thread_ip_action_call(p, t, action_id);
2106 }
2107
2108 static inline void
2109 instr_selector_exec(struct rte_swx_pipeline *p)
2110 {
2111         struct thread *t = &p->threads[p->thread_id];
2112         struct instruction *ip = t->ip;
2113         uint32_t selector_id = ip->table.table_id;
2114         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2115         struct selector_runtime *selector = &t->selectors[selector_id];
2116         struct selector_statistics *stats = &p->selector_stats[selector_id];
2117         uint64_t n_pkts = stats->n_pkts;
2118         int done;
2119
2120         /* Table. */
2121         done = rte_swx_table_selector_select(ts->obj,
2122                            selector->mailbox,
2123                            selector->group_id_buffer,
2124                            selector->selector_buffer,
2125                            selector->member_id_buffer);
2126         if (!done) {
2127                 /* Thread. */
2128                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2129                       p->thread_id,
2130                       selector_id);
2131
2132                 thread_yield(p);
2133                 return;
2134         }
2135
2136
2137         TRACE("[Thread %2u] selector %u\n",
2138               p->thread_id,
2139               selector_id);
2140
2141         stats->n_pkts = n_pkts + 1;
2142
2143         /* Thread. */
2144         thread_ip_inc(p);
2145 }
2146
2147 static inline void
2148 instr_learner_exec(struct rte_swx_pipeline *p)
2149 {
2150         struct thread *t = &p->threads[p->thread_id];
2151         struct instruction *ip = t->ip;
2152         uint32_t learner_id = ip->table.table_id;
2153         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2154                 p->n_selectors + learner_id];
2155         struct learner_runtime *l = &t->learners[learner_id];
2156         struct learner_statistics *stats = &p->learner_stats[learner_id];
2157         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2158         uint8_t *action_data;
2159         int done, hit;
2160
2161         /* Table. */
2162         time = rte_get_tsc_cycles();
2163
2164         done = rte_swx_table_learner_lookup(ts->obj,
2165                                             l->mailbox,
2166                                             time,
2167                                             l->key,
2168                                             &action_id,
2169                                             &action_data,
2170                                             &hit);
2171         if (!done) {
2172                 /* Thread. */
2173                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2174                       p->thread_id,
2175                       learner_id);
2176
2177                 thread_yield(p);
2178                 return;
2179         }
2180
2181         action_id = hit ? action_id : ts->default_action_id;
2182         action_data = hit ? action_data : ts->default_action_data;
2183         n_pkts_hit = stats->n_pkts_hit[hit];
2184         n_pkts_action = stats->n_pkts_action[action_id];
2185
2186         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2187               p->thread_id,
2188               learner_id,
2189               hit ? "hit" : "miss",
2190               (uint32_t)action_id);
2191
2192         t->action_id = action_id;
2193         t->structs[0] = action_data;
2194         t->hit = hit;
2195         t->learner_id = learner_id;
2196         t->time = time;
2197         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2198         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2199
2200         /* Thread. */
2201         thread_ip_action_call(p, t, action_id);
2202 }
2203
2204 /*
2205  * learn.
2206  */
2207 static struct action *
2208 action_find(struct rte_swx_pipeline *p, const char *name);
2209
2210 static int
2211 action_has_nbo_args(struct action *a);
2212
2213 static int
2214 instr_learn_translate(struct rte_swx_pipeline *p,
2215                       struct action *action,
2216                       char **tokens,
2217                       int n_tokens,
2218                       struct instruction *instr,
2219                       struct instruction_data *data __rte_unused)
2220 {
2221         struct action *a;
2222
2223         CHECK(action, EINVAL);
2224         CHECK(n_tokens == 2, EINVAL);
2225
2226         a = action_find(p, tokens[1]);
2227         CHECK(a, EINVAL);
2228         CHECK(!action_has_nbo_args(a), EINVAL);
2229
2230         instr->type = INSTR_LEARNER_LEARN;
2231         instr->learn.action_id = a->id;
2232
2233         return 0;
2234 }
2235
2236 static inline void
2237 instr_learn_exec(struct rte_swx_pipeline *p)
2238 {
2239         struct thread *t = &p->threads[p->thread_id];
2240         struct instruction *ip = t->ip;
2241         uint64_t action_id = ip->learn.action_id;
2242         uint32_t learner_id = t->learner_id;
2243         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2244                 p->n_selectors + learner_id];
2245         struct learner_runtime *l = &t->learners[learner_id];
2246         struct learner_statistics *stats = &p->learner_stats[learner_id];
2247         uint32_t status;
2248
2249         /* Table. */
2250         status = rte_swx_table_learner_add(ts->obj,
2251                                            l->mailbox,
2252                                            t->time,
2253                                            action_id,
2254                                            l->action_data[action_id]);
2255
2256         TRACE("[Thread %2u] learner %u learn %s\n",
2257               p->thread_id,
2258               learner_id,
2259               status ? "ok" : "error");
2260
2261         stats->n_pkts_learn[status] += 1;
2262
2263         /* Thread. */
2264         thread_ip_inc(p);
2265 }
2266
2267 /*
2268  * forget.
2269  */
2270 static int
2271 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2272                        struct action *action,
2273                        char **tokens __rte_unused,
2274                        int n_tokens,
2275                        struct instruction *instr,
2276                        struct instruction_data *data __rte_unused)
2277 {
2278         CHECK(action, EINVAL);
2279         CHECK(n_tokens == 1, EINVAL);
2280
2281         instr->type = INSTR_LEARNER_FORGET;
2282
2283         return 0;
2284 }
2285
2286 static inline void
2287 instr_forget_exec(struct rte_swx_pipeline *p)
2288 {
2289         struct thread *t = &p->threads[p->thread_id];
2290         uint32_t learner_id = t->learner_id;
2291         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2292                 p->n_selectors + learner_id];
2293         struct learner_runtime *l = &t->learners[learner_id];
2294         struct learner_statistics *stats = &p->learner_stats[learner_id];
2295
2296         /* Table. */
2297         rte_swx_table_learner_delete(ts->obj, l->mailbox);
2298
2299         TRACE("[Thread %2u] learner %u forget\n",
2300               p->thread_id,
2301               learner_id);
2302
2303         stats->n_pkts_forget += 1;
2304
2305         /* Thread. */
2306         thread_ip_inc(p);
2307 }
2308
2309 /*
2310  * extern.
2311  */
2312 static int
2313 instr_extern_translate(struct rte_swx_pipeline *p,
2314                        struct action *action __rte_unused,
2315                        char **tokens,
2316                        int n_tokens,
2317                        struct instruction *instr,
2318                        struct instruction_data *data __rte_unused)
2319 {
2320         char *token = tokens[1];
2321
2322         CHECK(n_tokens == 2, EINVAL);
2323
2324         if (token[0] == 'e') {
2325                 struct extern_obj *obj;
2326                 struct extern_type_member_func *func;
2327
2328                 func = extern_obj_member_func_parse(p, token, &obj);
2329                 CHECK(func, EINVAL);
2330
2331                 instr->type = INSTR_EXTERN_OBJ;
2332                 instr->ext_obj.ext_obj_id = obj->id;
2333                 instr->ext_obj.func_id = func->id;
2334
2335                 return 0;
2336         }
2337
2338         if (token[0] == 'f') {
2339                 struct extern_func *func;
2340
2341                 func = extern_func_parse(p, token);
2342                 CHECK(func, EINVAL);
2343
2344                 instr->type = INSTR_EXTERN_FUNC;
2345                 instr->ext_func.ext_func_id = func->id;
2346
2347                 return 0;
2348         }
2349
2350         CHECK(0, EINVAL);
2351 }
2352
2353 static inline void
2354 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2355 {
2356         struct thread *t = &p->threads[p->thread_id];
2357         struct instruction *ip = t->ip;
2358         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2359         uint32_t func_id = ip->ext_obj.func_id;
2360         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2361         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2362
2363         TRACE("[Thread %2u] extern obj %u member func %u\n",
2364               p->thread_id,
2365               obj_id,
2366               func_id);
2367
2368         /* Extern object member function execute. */
2369         uint32_t done = func(obj->obj, obj->mailbox);
2370
2371         /* Thread. */
2372         thread_ip_inc_cond(t, done);
2373         thread_yield_cond(p, done ^ 1);
2374 }
2375
2376 static inline void
2377 instr_extern_func_exec(struct rte_swx_pipeline *p)
2378 {
2379         struct thread *t = &p->threads[p->thread_id];
2380         struct instruction *ip = t->ip;
2381         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2382         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2383         rte_swx_extern_func_t func = ext_func->func;
2384
2385         TRACE("[Thread %2u] extern func %u\n",
2386               p->thread_id,
2387               ext_func_id);
2388
2389         /* Extern function execute. */
2390         uint32_t done = func(ext_func->mailbox);
2391
2392         /* Thread. */
2393         thread_ip_inc_cond(t, done);
2394         thread_yield_cond(p, done ^ 1);
2395 }
2396
2397 /*
2398  * mov.
2399  */
2400 static int
2401 instr_mov_translate(struct rte_swx_pipeline *p,
2402                     struct action *action,
2403                     char **tokens,
2404                     int n_tokens,
2405                     struct instruction *instr,
2406                     struct instruction_data *data __rte_unused)
2407 {
2408         char *dst = tokens[1], *src = tokens[2];
2409         struct field *fdst, *fsrc;
2410         uint64_t src_val;
2411         uint32_t dst_struct_id = 0, src_struct_id = 0;
2412
2413         CHECK(n_tokens == 3, EINVAL);
2414
2415         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2416         CHECK(fdst, EINVAL);
2417         CHECK(!fdst->var_size, EINVAL);
2418
2419         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2420         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2421         if (fsrc) {
2422                 CHECK(!fsrc->var_size, EINVAL);
2423
2424                 instr->type = INSTR_MOV;
2425                 if (dst[0] != 'h' && src[0] == 'h')
2426                         instr->type = INSTR_MOV_MH;
2427                 if (dst[0] == 'h' && src[0] != 'h')
2428                         instr->type = INSTR_MOV_HM;
2429                 if (dst[0] == 'h' && src[0] == 'h')
2430                         instr->type = INSTR_MOV_HH;
2431
2432                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2433                 instr->mov.dst.n_bits = fdst->n_bits;
2434                 instr->mov.dst.offset = fdst->offset / 8;
2435                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2436                 instr->mov.src.n_bits = fsrc->n_bits;
2437                 instr->mov.src.offset = fsrc->offset / 8;
2438                 return 0;
2439         }
2440
2441         /* MOV_I. */
2442         src_val = strtoull(src, &src, 0);
2443         CHECK(!src[0], EINVAL);
2444
2445         if (dst[0] == 'h')
2446                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2447
2448         instr->type = INSTR_MOV_I;
2449         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2450         instr->mov.dst.n_bits = fdst->n_bits;
2451         instr->mov.dst.offset = fdst->offset / 8;
2452         instr->mov.src_val = src_val;
2453         return 0;
2454 }
2455
2456 static inline void
2457 instr_mov_exec(struct rte_swx_pipeline *p)
2458 {
2459         struct thread *t = &p->threads[p->thread_id];
2460         struct instruction *ip = t->ip;
2461
2462         TRACE("[Thread %2u] mov\n",
2463               p->thread_id);
2464
2465         MOV(t, ip);
2466
2467         /* Thread. */
2468         thread_ip_inc(p);
2469 }
2470
2471 static inline void
2472 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2473 {
2474         struct thread *t = &p->threads[p->thread_id];
2475         struct instruction *ip = t->ip;
2476
2477         TRACE("[Thread %2u] mov (mh)\n",
2478               p->thread_id);
2479
2480         MOV_MH(t, ip);
2481
2482         /* Thread. */
2483         thread_ip_inc(p);
2484 }
2485
2486 static inline void
2487 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2488 {
2489         struct thread *t = &p->threads[p->thread_id];
2490         struct instruction *ip = t->ip;
2491
2492         TRACE("[Thread %2u] mov (hm)\n",
2493               p->thread_id);
2494
2495         MOV_HM(t, ip);
2496
2497         /* Thread. */
2498         thread_ip_inc(p);
2499 }
2500
2501 static inline void
2502 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2503 {
2504         struct thread *t = &p->threads[p->thread_id];
2505         struct instruction *ip = t->ip;
2506
2507         TRACE("[Thread %2u] mov (hh)\n",
2508               p->thread_id);
2509
2510         MOV_HH(t, ip);
2511
2512         /* Thread. */
2513         thread_ip_inc(p);
2514 }
2515
2516 static inline void
2517 instr_mov_i_exec(struct rte_swx_pipeline *p)
2518 {
2519         struct thread *t = &p->threads[p->thread_id];
2520         struct instruction *ip = t->ip;
2521
2522         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
2523               p->thread_id,
2524               ip->mov.src_val);
2525
2526         MOV_I(t, ip);
2527
2528         /* Thread. */
2529         thread_ip_inc(p);
2530 }
2531
2532 /*
2533  * dma.
2534  */
2535 static inline void
2536 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2537
2538 static inline void
2539 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2540 {
2541         struct thread *t = &p->threads[p->thread_id];
2542         struct instruction *ip = t->ip;
2543         uint8_t *action_data = t->structs[0];
2544         uint64_t valid_headers = t->valid_headers;
2545         uint32_t i;
2546
2547         for (i = 0; i < n_dma; i++) {
2548                 uint32_t header_id = ip->dma.dst.header_id[i];
2549                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2550                 uint32_t offset = ip->dma.src.offset[i];
2551                 uint32_t n_bytes = ip->dma.n_bytes[i];
2552
2553                 struct header_runtime *h = &t->headers[header_id];
2554                 uint8_t *h_ptr0 = h->ptr0;
2555                 uint8_t *h_ptr = t->structs[struct_id];
2556
2557                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2558                         h_ptr : h_ptr0;
2559                 void *src = &action_data[offset];
2560
2561                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2562
2563                 /* Headers. */
2564                 memcpy(dst, src, n_bytes);
2565                 t->structs[struct_id] = dst;
2566                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2567         }
2568
2569         t->valid_headers = valid_headers;
2570 }
2571
2572 static inline void
2573 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2574 {
2575         __instr_dma_ht_exec(p, 1);
2576
2577         /* Thread. */
2578         thread_ip_inc(p);
2579 }
2580
2581 static inline void
2582 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2583 {
2584         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2585               p->thread_id);
2586
2587         __instr_dma_ht_exec(p, 2);
2588
2589         /* Thread. */
2590         thread_ip_inc(p);
2591 }
2592
2593 static inline void
2594 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2595 {
2596         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2597               p->thread_id);
2598
2599         __instr_dma_ht_exec(p, 3);
2600
2601         /* Thread. */
2602         thread_ip_inc(p);
2603 }
2604
2605 static inline void
2606 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2607 {
2608         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2609               p->thread_id);
2610
2611         __instr_dma_ht_exec(p, 4);
2612
2613         /* Thread. */
2614         thread_ip_inc(p);
2615 }
2616
2617 static inline void
2618 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2619 {
2620         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2621               p->thread_id);
2622
2623         __instr_dma_ht_exec(p, 5);
2624
2625         /* Thread. */
2626         thread_ip_inc(p);
2627 }
2628
2629 static inline void
2630 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2631 {
2632         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2633               p->thread_id);
2634
2635         __instr_dma_ht_exec(p, 6);
2636
2637         /* Thread. */
2638         thread_ip_inc(p);
2639 }
2640
2641 static inline void
2642 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2643 {
2644         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2645               p->thread_id);
2646
2647         __instr_dma_ht_exec(p, 7);
2648
2649         /* Thread. */
2650         thread_ip_inc(p);
2651 }
2652
2653 static inline void
2654 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2655 {
2656         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2657               p->thread_id);
2658
2659         __instr_dma_ht_exec(p, 8);
2660
2661         /* Thread. */
2662         thread_ip_inc(p);
2663 }
2664
2665 /*
2666  * alu.
2667  */
2668 static int
2669 instr_alu_add_translate(struct rte_swx_pipeline *p,
2670                         struct action *action,
2671                         char **tokens,
2672                         int n_tokens,
2673                         struct instruction *instr,
2674                         struct instruction_data *data __rte_unused)
2675 {
2676         char *dst = tokens[1], *src = tokens[2];
2677         struct field *fdst, *fsrc;
2678         uint64_t src_val;
2679         uint32_t dst_struct_id = 0, src_struct_id = 0;
2680
2681         CHECK(n_tokens == 3, EINVAL);
2682
2683         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2684         CHECK(fdst, EINVAL);
2685         CHECK(!fdst->var_size, EINVAL);
2686
2687         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2688         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2689         if (fsrc) {
2690                 CHECK(!fsrc->var_size, EINVAL);
2691
2692                 instr->type = INSTR_ALU_ADD;
2693                 if (dst[0] == 'h' && src[0] != 'h')
2694                         instr->type = INSTR_ALU_ADD_HM;
2695                 if (dst[0] != 'h' && src[0] == 'h')
2696                         instr->type = INSTR_ALU_ADD_MH;
2697                 if (dst[0] == 'h' && src[0] == 'h')
2698                         instr->type = INSTR_ALU_ADD_HH;
2699
2700                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2701                 instr->alu.dst.n_bits = fdst->n_bits;
2702                 instr->alu.dst.offset = fdst->offset / 8;
2703                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2704                 instr->alu.src.n_bits = fsrc->n_bits;
2705                 instr->alu.src.offset = fsrc->offset / 8;
2706                 return 0;
2707         }
2708
2709         /* ADD_MI, ADD_HI. */
2710         src_val = strtoull(src, &src, 0);
2711         CHECK(!src[0], EINVAL);
2712
2713         instr->type = INSTR_ALU_ADD_MI;
2714         if (dst[0] == 'h')
2715                 instr->type = INSTR_ALU_ADD_HI;
2716
2717         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2718         instr->alu.dst.n_bits = fdst->n_bits;
2719         instr->alu.dst.offset = fdst->offset / 8;
2720         instr->alu.src_val = src_val;
2721         return 0;
2722 }
2723
2724 static int
2725 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2726                         struct action *action,
2727                         char **tokens,
2728                         int n_tokens,
2729                         struct instruction *instr,
2730                         struct instruction_data *data __rte_unused)
2731 {
2732         char *dst = tokens[1], *src = tokens[2];
2733         struct field *fdst, *fsrc;
2734         uint64_t src_val;
2735         uint32_t dst_struct_id = 0, src_struct_id = 0;
2736
2737         CHECK(n_tokens == 3, EINVAL);
2738
2739         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2740         CHECK(fdst, EINVAL);
2741         CHECK(!fdst->var_size, EINVAL);
2742
2743         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2744         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2745         if (fsrc) {
2746                 CHECK(!fsrc->var_size, EINVAL);
2747
2748                 instr->type = INSTR_ALU_SUB;
2749                 if (dst[0] == 'h' && src[0] != 'h')
2750                         instr->type = INSTR_ALU_SUB_HM;
2751                 if (dst[0] != 'h' && src[0] == 'h')
2752                         instr->type = INSTR_ALU_SUB_MH;
2753                 if (dst[0] == 'h' && src[0] == 'h')
2754                         instr->type = INSTR_ALU_SUB_HH;
2755
2756                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2757                 instr->alu.dst.n_bits = fdst->n_bits;
2758                 instr->alu.dst.offset = fdst->offset / 8;
2759                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2760                 instr->alu.src.n_bits = fsrc->n_bits;
2761                 instr->alu.src.offset = fsrc->offset / 8;
2762                 return 0;
2763         }
2764
2765         /* SUB_MI, SUB_HI. */
2766         src_val = strtoull(src, &src, 0);
2767         CHECK(!src[0], EINVAL);
2768
2769         instr->type = INSTR_ALU_SUB_MI;
2770         if (dst[0] == 'h')
2771                 instr->type = INSTR_ALU_SUB_HI;
2772
2773         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2774         instr->alu.dst.n_bits = fdst->n_bits;
2775         instr->alu.dst.offset = fdst->offset / 8;
2776         instr->alu.src_val = src_val;
2777         return 0;
2778 }
2779
2780 static int
2781 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2782                           struct action *action __rte_unused,
2783                           char **tokens,
2784                           int n_tokens,
2785                           struct instruction *instr,
2786                           struct instruction_data *data __rte_unused)
2787 {
2788         char *dst = tokens[1], *src = tokens[2];
2789         struct header *hdst, *hsrc;
2790         struct field *fdst, *fsrc;
2791
2792         CHECK(n_tokens == 3, EINVAL);
2793
2794         fdst = header_field_parse(p, dst, &hdst);
2795         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2796         CHECK(!fdst->var_size, EINVAL);
2797
2798         /* CKADD_FIELD. */
2799         fsrc = header_field_parse(p, src, &hsrc);
2800         if (fsrc) {
2801                 CHECK(!fsrc->var_size, EINVAL);
2802
2803                 instr->type = INSTR_ALU_CKADD_FIELD;
2804                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2805                 instr->alu.dst.n_bits = fdst->n_bits;
2806                 instr->alu.dst.offset = fdst->offset / 8;
2807                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2808                 instr->alu.src.n_bits = fsrc->n_bits;
2809                 instr->alu.src.offset = fsrc->offset / 8;
2810                 return 0;
2811         }
2812
2813         /* CKADD_STRUCT, CKADD_STRUCT20. */
2814         hsrc = header_parse(p, src);
2815         CHECK(hsrc, EINVAL);
2816         CHECK(!hsrc->st->var_size, EINVAL);
2817
2818         instr->type = INSTR_ALU_CKADD_STRUCT;
2819         if ((hsrc->st->n_bits / 8) == 20)
2820                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2821
2822         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2823         instr->alu.dst.n_bits = fdst->n_bits;
2824         instr->alu.dst.offset = fdst->offset / 8;
2825         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2826         instr->alu.src.n_bits = hsrc->st->n_bits;
2827         instr->alu.src.offset = 0; /* Unused. */
2828         return 0;
2829 }
2830
2831 static int
2832 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2833                           struct action *action __rte_unused,
2834                           char **tokens,
2835                           int n_tokens,
2836                           struct instruction *instr,
2837                           struct instruction_data *data __rte_unused)
2838 {
2839         char *dst = tokens[1], *src = tokens[2];
2840         struct header *hdst, *hsrc;
2841         struct field *fdst, *fsrc;
2842
2843         CHECK(n_tokens == 3, EINVAL);
2844
2845         fdst = header_field_parse(p, dst, &hdst);
2846         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2847         CHECK(!fdst->var_size, EINVAL);
2848
2849         fsrc = header_field_parse(p, src, &hsrc);
2850         CHECK(fsrc, EINVAL);
2851         CHECK(!fsrc->var_size, EINVAL);
2852
2853         instr->type = INSTR_ALU_CKSUB_FIELD;
2854         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2855         instr->alu.dst.n_bits = fdst->n_bits;
2856         instr->alu.dst.offset = fdst->offset / 8;
2857         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2858         instr->alu.src.n_bits = fsrc->n_bits;
2859         instr->alu.src.offset = fsrc->offset / 8;
2860         return 0;
2861 }
2862
2863 static int
2864 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2865                         struct action *action,
2866                         char **tokens,
2867                         int n_tokens,
2868                         struct instruction *instr,
2869                         struct instruction_data *data __rte_unused)
2870 {
2871         char *dst = tokens[1], *src = tokens[2];
2872         struct field *fdst, *fsrc;
2873         uint64_t src_val;
2874         uint32_t dst_struct_id = 0, src_struct_id = 0;
2875
2876         CHECK(n_tokens == 3, EINVAL);
2877
2878         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2879         CHECK(fdst, EINVAL);
2880         CHECK(!fdst->var_size, EINVAL);
2881
2882         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2883         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2884         if (fsrc) {
2885                 CHECK(!fsrc->var_size, EINVAL);
2886
2887                 instr->type = INSTR_ALU_SHL;
2888                 if (dst[0] == 'h' && src[0] != 'h')
2889                         instr->type = INSTR_ALU_SHL_HM;
2890                 if (dst[0] != 'h' && src[0] == 'h')
2891                         instr->type = INSTR_ALU_SHL_MH;
2892                 if (dst[0] == 'h' && src[0] == 'h')
2893                         instr->type = INSTR_ALU_SHL_HH;
2894
2895                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2896                 instr->alu.dst.n_bits = fdst->n_bits;
2897                 instr->alu.dst.offset = fdst->offset / 8;
2898                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2899                 instr->alu.src.n_bits = fsrc->n_bits;
2900                 instr->alu.src.offset = fsrc->offset / 8;
2901                 return 0;
2902         }
2903
2904         /* SHL_MI, SHL_HI. */
2905         src_val = strtoull(src, &src, 0);
2906         CHECK(!src[0], EINVAL);
2907
2908         instr->type = INSTR_ALU_SHL_MI;
2909         if (dst[0] == 'h')
2910                 instr->type = INSTR_ALU_SHL_HI;
2911
2912         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2913         instr->alu.dst.n_bits = fdst->n_bits;
2914         instr->alu.dst.offset = fdst->offset / 8;
2915         instr->alu.src_val = src_val;
2916         return 0;
2917 }
2918
2919 static int
2920 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2921                         struct action *action,
2922                         char **tokens,
2923                         int n_tokens,
2924                         struct instruction *instr,
2925                         struct instruction_data *data __rte_unused)
2926 {
2927         char *dst = tokens[1], *src = tokens[2];
2928         struct field *fdst, *fsrc;
2929         uint64_t src_val;
2930         uint32_t dst_struct_id = 0, src_struct_id = 0;
2931
2932         CHECK(n_tokens == 3, EINVAL);
2933
2934         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2935         CHECK(fdst, EINVAL);
2936         CHECK(!fdst->var_size, EINVAL);
2937
2938         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2939         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2940         if (fsrc) {
2941                 CHECK(!fsrc->var_size, EINVAL);
2942
2943                 instr->type = INSTR_ALU_SHR;
2944                 if (dst[0] == 'h' && src[0] != 'h')
2945                         instr->type = INSTR_ALU_SHR_HM;
2946                 if (dst[0] != 'h' && src[0] == 'h')
2947                         instr->type = INSTR_ALU_SHR_MH;
2948                 if (dst[0] == 'h' && src[0] == 'h')
2949                         instr->type = INSTR_ALU_SHR_HH;
2950
2951                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2952                 instr->alu.dst.n_bits = fdst->n_bits;
2953                 instr->alu.dst.offset = fdst->offset / 8;
2954                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2955                 instr->alu.src.n_bits = fsrc->n_bits;
2956                 instr->alu.src.offset = fsrc->offset / 8;
2957                 return 0;
2958         }
2959
2960         /* SHR_MI, SHR_HI. */
2961         src_val = strtoull(src, &src, 0);
2962         CHECK(!src[0], EINVAL);
2963
2964         instr->type = INSTR_ALU_SHR_MI;
2965         if (dst[0] == 'h')
2966                 instr->type = INSTR_ALU_SHR_HI;
2967
2968         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2969         instr->alu.dst.n_bits = fdst->n_bits;
2970         instr->alu.dst.offset = fdst->offset / 8;
2971         instr->alu.src_val = src_val;
2972         return 0;
2973 }
2974
2975 static int
2976 instr_alu_and_translate(struct rte_swx_pipeline *p,
2977                         struct action *action,
2978                         char **tokens,
2979                         int n_tokens,
2980                         struct instruction *instr,
2981                         struct instruction_data *data __rte_unused)
2982 {
2983         char *dst = tokens[1], *src = tokens[2];
2984         struct field *fdst, *fsrc;
2985         uint64_t src_val;
2986         uint32_t dst_struct_id = 0, src_struct_id = 0;
2987
2988         CHECK(n_tokens == 3, EINVAL);
2989
2990         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2991         CHECK(fdst, EINVAL);
2992         CHECK(!fdst->var_size, EINVAL);
2993
2994         /* AND, AND_MH, AND_HM, AND_HH. */
2995         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2996         if (fsrc) {
2997                 CHECK(!fsrc->var_size, EINVAL);
2998
2999                 instr->type = INSTR_ALU_AND;
3000                 if (dst[0] != 'h' && src[0] == 'h')
3001                         instr->type = INSTR_ALU_AND_MH;
3002                 if (dst[0] == 'h' && src[0] != 'h')
3003                         instr->type = INSTR_ALU_AND_HM;
3004                 if (dst[0] == 'h' && src[0] == 'h')
3005                         instr->type = INSTR_ALU_AND_HH;
3006
3007                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3008                 instr->alu.dst.n_bits = fdst->n_bits;
3009                 instr->alu.dst.offset = fdst->offset / 8;
3010                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3011                 instr->alu.src.n_bits = fsrc->n_bits;
3012                 instr->alu.src.offset = fsrc->offset / 8;
3013                 return 0;
3014         }
3015
3016         /* AND_I. */
3017         src_val = strtoull(src, &src, 0);
3018         CHECK(!src[0], EINVAL);
3019
3020         if (dst[0] == 'h')
3021                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3022
3023         instr->type = INSTR_ALU_AND_I;
3024         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025         instr->alu.dst.n_bits = fdst->n_bits;
3026         instr->alu.dst.offset = fdst->offset / 8;
3027         instr->alu.src_val = src_val;
3028         return 0;
3029 }
3030
3031 static int
3032 instr_alu_or_translate(struct rte_swx_pipeline *p,
3033                        struct action *action,
3034                        char **tokens,
3035                        int n_tokens,
3036                        struct instruction *instr,
3037                        struct instruction_data *data __rte_unused)
3038 {
3039         char *dst = tokens[1], *src = tokens[2];
3040         struct field *fdst, *fsrc;
3041         uint64_t src_val;
3042         uint32_t dst_struct_id = 0, src_struct_id = 0;
3043
3044         CHECK(n_tokens == 3, EINVAL);
3045
3046         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3047         CHECK(fdst, EINVAL);
3048         CHECK(!fdst->var_size, EINVAL);
3049
3050         /* OR, OR_MH, OR_HM, OR_HH. */
3051         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3052         if (fsrc) {
3053                 CHECK(!fsrc->var_size, EINVAL);
3054
3055                 instr->type = INSTR_ALU_OR;
3056                 if (dst[0] != 'h' && src[0] == 'h')
3057                         instr->type = INSTR_ALU_OR_MH;
3058                 if (dst[0] == 'h' && src[0] != 'h')
3059                         instr->type = INSTR_ALU_OR_HM;
3060                 if (dst[0] == 'h' && src[0] == 'h')
3061                         instr->type = INSTR_ALU_OR_HH;
3062
3063                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3064                 instr->alu.dst.n_bits = fdst->n_bits;
3065                 instr->alu.dst.offset = fdst->offset / 8;
3066                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3067                 instr->alu.src.n_bits = fsrc->n_bits;
3068                 instr->alu.src.offset = fsrc->offset / 8;
3069                 return 0;
3070         }
3071
3072         /* OR_I. */
3073         src_val = strtoull(src, &src, 0);
3074         CHECK(!src[0], EINVAL);
3075
3076         if (dst[0] == 'h')
3077                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3078
3079         instr->type = INSTR_ALU_OR_I;
3080         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3081         instr->alu.dst.n_bits = fdst->n_bits;
3082         instr->alu.dst.offset = fdst->offset / 8;
3083         instr->alu.src_val = src_val;
3084         return 0;
3085 }
3086
3087 static int
3088 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3089                         struct action *action,
3090                         char **tokens,
3091                         int n_tokens,
3092                         struct instruction *instr,
3093                         struct instruction_data *data __rte_unused)
3094 {
3095         char *dst = tokens[1], *src = tokens[2];
3096         struct field *fdst, *fsrc;
3097         uint64_t src_val;
3098         uint32_t dst_struct_id = 0, src_struct_id = 0;
3099
3100         CHECK(n_tokens == 3, EINVAL);
3101
3102         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3103         CHECK(fdst, EINVAL);
3104         CHECK(!fdst->var_size, EINVAL);
3105
3106         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3107         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3108         if (fsrc) {
3109                 CHECK(!fsrc->var_size, EINVAL);
3110
3111                 instr->type = INSTR_ALU_XOR;
3112                 if (dst[0] != 'h' && src[0] == 'h')
3113                         instr->type = INSTR_ALU_XOR_MH;
3114                 if (dst[0] == 'h' && src[0] != 'h')
3115                         instr->type = INSTR_ALU_XOR_HM;
3116                 if (dst[0] == 'h' && src[0] == 'h')
3117                         instr->type = INSTR_ALU_XOR_HH;
3118
3119                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3120                 instr->alu.dst.n_bits = fdst->n_bits;
3121                 instr->alu.dst.offset = fdst->offset / 8;
3122                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3123                 instr->alu.src.n_bits = fsrc->n_bits;
3124                 instr->alu.src.offset = fsrc->offset / 8;
3125                 return 0;
3126         }
3127
3128         /* XOR_I. */
3129         src_val = strtoull(src, &src, 0);
3130         CHECK(!src[0], EINVAL);
3131
3132         if (dst[0] == 'h')
3133                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3134
3135         instr->type = INSTR_ALU_XOR_I;
3136         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137         instr->alu.dst.n_bits = fdst->n_bits;
3138         instr->alu.dst.offset = fdst->offset / 8;
3139         instr->alu.src_val = src_val;
3140         return 0;
3141 }
3142
3143 static inline void
3144 instr_alu_add_exec(struct rte_swx_pipeline *p)
3145 {
3146         struct thread *t = &p->threads[p->thread_id];
3147         struct instruction *ip = t->ip;
3148
3149         TRACE("[Thread %2u] add\n", p->thread_id);
3150
3151         /* Structs. */
3152         ALU(t, ip, +);
3153
3154         /* Thread. */
3155         thread_ip_inc(p);
3156 }
3157
3158 static inline void
3159 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3160 {
3161         struct thread *t = &p->threads[p->thread_id];
3162         struct instruction *ip = t->ip;
3163
3164         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3165
3166         /* Structs. */
3167         ALU_MH(t, ip, +);
3168
3169         /* Thread. */
3170         thread_ip_inc(p);
3171 }
3172
3173 static inline void
3174 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3175 {
3176         struct thread *t = &p->threads[p->thread_id];
3177         struct instruction *ip = t->ip;
3178
3179         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3180
3181         /* Structs. */
3182         ALU_HM(t, ip, +);
3183
3184         /* Thread. */
3185         thread_ip_inc(p);
3186 }
3187
3188 static inline void
3189 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3190 {
3191         struct thread *t = &p->threads[p->thread_id];
3192         struct instruction *ip = t->ip;
3193
3194         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3195
3196         /* Structs. */
3197         ALU_HH(t, ip, +);
3198
3199         /* Thread. */
3200         thread_ip_inc(p);
3201 }
3202
3203 static inline void
3204 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3205 {
3206         struct thread *t = &p->threads[p->thread_id];
3207         struct instruction *ip = t->ip;
3208
3209         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3210
3211         /* Structs. */
3212         ALU_MI(t, ip, +);
3213
3214         /* Thread. */
3215         thread_ip_inc(p);
3216 }
3217
3218 static inline void
3219 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3220 {
3221         struct thread *t = &p->threads[p->thread_id];
3222         struct instruction *ip = t->ip;
3223
3224         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3225
3226         /* Structs. */
3227         ALU_HI(t, ip, +);
3228
3229         /* Thread. */
3230         thread_ip_inc(p);
3231 }
3232
3233 static inline void
3234 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3235 {
3236         struct thread *t = &p->threads[p->thread_id];
3237         struct instruction *ip = t->ip;
3238
3239         TRACE("[Thread %2u] sub\n", p->thread_id);
3240
3241         /* Structs. */
3242         ALU(t, ip, -);
3243
3244         /* Thread. */
3245         thread_ip_inc(p);
3246 }
3247
3248 static inline void
3249 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3250 {
3251         struct thread *t = &p->threads[p->thread_id];
3252         struct instruction *ip = t->ip;
3253
3254         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3255
3256         /* Structs. */
3257         ALU_MH(t, ip, -);
3258
3259         /* Thread. */
3260         thread_ip_inc(p);
3261 }
3262
3263 static inline void
3264 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3265 {
3266         struct thread *t = &p->threads[p->thread_id];
3267         struct instruction *ip = t->ip;
3268
3269         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3270
3271         /* Structs. */
3272         ALU_HM(t, ip, -);
3273
3274         /* Thread. */
3275         thread_ip_inc(p);
3276 }
3277
3278 static inline void
3279 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3280 {
3281         struct thread *t = &p->threads[p->thread_id];
3282         struct instruction *ip = t->ip;
3283
3284         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3285
3286         /* Structs. */
3287         ALU_HH(t, ip, -);
3288
3289         /* Thread. */
3290         thread_ip_inc(p);
3291 }
3292
3293 static inline void
3294 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3295 {
3296         struct thread *t = &p->threads[p->thread_id];
3297         struct instruction *ip = t->ip;
3298
3299         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3300
3301         /* Structs. */
3302         ALU_MI(t, ip, -);
3303
3304         /* Thread. */
3305         thread_ip_inc(p);
3306 }
3307
3308 static inline void
3309 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3310 {
3311         struct thread *t = &p->threads[p->thread_id];
3312         struct instruction *ip = t->ip;
3313
3314         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3315
3316         /* Structs. */
3317         ALU_HI(t, ip, -);
3318
3319         /* Thread. */
3320         thread_ip_inc(p);
3321 }
3322
3323 static inline void
3324 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3325 {
3326         struct thread *t = &p->threads[p->thread_id];
3327         struct instruction *ip = t->ip;
3328
3329         TRACE("[Thread %2u] shl\n", p->thread_id);
3330
3331         /* Structs. */
3332         ALU(t, ip, <<);
3333
3334         /* Thread. */
3335         thread_ip_inc(p);
3336 }
3337
3338 static inline void
3339 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3340 {
3341         struct thread *t = &p->threads[p->thread_id];
3342         struct instruction *ip = t->ip;
3343
3344         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3345
3346         /* Structs. */
3347         ALU_MH(t, ip, <<);
3348
3349         /* Thread. */
3350         thread_ip_inc(p);
3351 }
3352
3353 static inline void
3354 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3355 {
3356         struct thread *t = &p->threads[p->thread_id];
3357         struct instruction *ip = t->ip;
3358
3359         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3360
3361         /* Structs. */
3362         ALU_HM(t, ip, <<);
3363
3364         /* Thread. */
3365         thread_ip_inc(p);
3366 }
3367
3368 static inline void
3369 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3370 {
3371         struct thread *t = &p->threads[p->thread_id];
3372         struct instruction *ip = t->ip;
3373
3374         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3375
3376         /* Structs. */
3377         ALU_HH(t, ip, <<);
3378
3379         /* Thread. */
3380         thread_ip_inc(p);
3381 }
3382
3383 static inline void
3384 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3385 {
3386         struct thread *t = &p->threads[p->thread_id];
3387         struct instruction *ip = t->ip;
3388
3389         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3390
3391         /* Structs. */
3392         ALU_MI(t, ip, <<);
3393
3394         /* Thread. */
3395         thread_ip_inc(p);
3396 }
3397
3398 static inline void
3399 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3400 {
3401         struct thread *t = &p->threads[p->thread_id];
3402         struct instruction *ip = t->ip;
3403
3404         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3405
3406         /* Structs. */
3407         ALU_HI(t, ip, <<);
3408
3409         /* Thread. */
3410         thread_ip_inc(p);
3411 }
3412
3413 static inline void
3414 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3415 {
3416         struct thread *t = &p->threads[p->thread_id];
3417         struct instruction *ip = t->ip;
3418
3419         TRACE("[Thread %2u] shr\n", p->thread_id);
3420
3421         /* Structs. */
3422         ALU(t, ip, >>);
3423
3424         /* Thread. */
3425         thread_ip_inc(p);
3426 }
3427
3428 static inline void
3429 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3430 {
3431         struct thread *t = &p->threads[p->thread_id];
3432         struct instruction *ip = t->ip;
3433
3434         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3435
3436         /* Structs. */
3437         ALU_MH(t, ip, >>);
3438
3439         /* Thread. */
3440         thread_ip_inc(p);
3441 }
3442
3443 static inline void
3444 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3445 {
3446         struct thread *t = &p->threads[p->thread_id];
3447         struct instruction *ip = t->ip;
3448
3449         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3450
3451         /* Structs. */
3452         ALU_HM(t, ip, >>);
3453
3454         /* Thread. */
3455         thread_ip_inc(p);
3456 }
3457
3458 static inline void
3459 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3460 {
3461         struct thread *t = &p->threads[p->thread_id];
3462         struct instruction *ip = t->ip;
3463
3464         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3465
3466         /* Structs. */
3467         ALU_HH(t, ip, >>);
3468
3469         /* Thread. */
3470         thread_ip_inc(p);
3471 }
3472
3473 static inline void
3474 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3475 {
3476         struct thread *t = &p->threads[p->thread_id];
3477         struct instruction *ip = t->ip;
3478
3479         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3480
3481         /* Structs. */
3482         ALU_MI(t, ip, >>);
3483
3484         /* Thread. */
3485         thread_ip_inc(p);
3486 }
3487
3488 static inline void
3489 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3490 {
3491         struct thread *t = &p->threads[p->thread_id];
3492         struct instruction *ip = t->ip;
3493
3494         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3495
3496         /* Structs. */
3497         ALU_HI(t, ip, >>);
3498
3499         /* Thread. */
3500         thread_ip_inc(p);
3501 }
3502
3503 static inline void
3504 instr_alu_and_exec(struct rte_swx_pipeline *p)
3505 {
3506         struct thread *t = &p->threads[p->thread_id];
3507         struct instruction *ip = t->ip;
3508
3509         TRACE("[Thread %2u] and\n", p->thread_id);
3510
3511         /* Structs. */
3512         ALU(t, ip, &);
3513
3514         /* Thread. */
3515         thread_ip_inc(p);
3516 }
3517
3518 static inline void
3519 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3520 {
3521         struct thread *t = &p->threads[p->thread_id];
3522         struct instruction *ip = t->ip;
3523
3524         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3525
3526         /* Structs. */
3527         ALU_MH(t, ip, &);
3528
3529         /* Thread. */
3530         thread_ip_inc(p);
3531 }
3532
3533 static inline void
3534 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3535 {
3536         struct thread *t = &p->threads[p->thread_id];
3537         struct instruction *ip = t->ip;
3538
3539         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3540
3541         /* Structs. */
3542         ALU_HM_FAST(t, ip, &);
3543
3544         /* Thread. */
3545         thread_ip_inc(p);
3546 }
3547
3548 static inline void
3549 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3550 {
3551         struct thread *t = &p->threads[p->thread_id];
3552         struct instruction *ip = t->ip;
3553
3554         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3555
3556         /* Structs. */
3557         ALU_HH_FAST(t, ip, &);
3558
3559         /* Thread. */
3560         thread_ip_inc(p);
3561 }
3562
3563 static inline void
3564 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3565 {
3566         struct thread *t = &p->threads[p->thread_id];
3567         struct instruction *ip = t->ip;
3568
3569         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3570
3571         /* Structs. */
3572         ALU_I(t, ip, &);
3573
3574         /* Thread. */
3575         thread_ip_inc(p);
3576 }
3577
3578 static inline void
3579 instr_alu_or_exec(struct rte_swx_pipeline *p)
3580 {
3581         struct thread *t = &p->threads[p->thread_id];
3582         struct instruction *ip = t->ip;
3583
3584         TRACE("[Thread %2u] or\n", p->thread_id);
3585
3586         /* Structs. */
3587         ALU(t, ip, |);
3588
3589         /* Thread. */
3590         thread_ip_inc(p);
3591 }
3592
3593 static inline void
3594 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3595 {
3596         struct thread *t = &p->threads[p->thread_id];
3597         struct instruction *ip = t->ip;
3598
3599         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3600
3601         /* Structs. */
3602         ALU_MH(t, ip, |);
3603
3604         /* Thread. */
3605         thread_ip_inc(p);
3606 }
3607
3608 static inline void
3609 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3610 {
3611         struct thread *t = &p->threads[p->thread_id];
3612         struct instruction *ip = t->ip;
3613
3614         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3615
3616         /* Structs. */
3617         ALU_HM_FAST(t, ip, |);
3618
3619         /* Thread. */
3620         thread_ip_inc(p);
3621 }
3622
3623 static inline void
3624 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3625 {
3626         struct thread *t = &p->threads[p->thread_id];
3627         struct instruction *ip = t->ip;
3628
3629         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3630
3631         /* Structs. */
3632         ALU_HH_FAST(t, ip, |);
3633
3634         /* Thread. */
3635         thread_ip_inc(p);
3636 }
3637
3638 static inline void
3639 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3640 {
3641         struct thread *t = &p->threads[p->thread_id];
3642         struct instruction *ip = t->ip;
3643
3644         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3645
3646         /* Structs. */
3647         ALU_I(t, ip, |);
3648
3649         /* Thread. */
3650         thread_ip_inc(p);
3651 }
3652
3653 static inline void
3654 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3655 {
3656         struct thread *t = &p->threads[p->thread_id];
3657         struct instruction *ip = t->ip;
3658
3659         TRACE("[Thread %2u] xor\n", p->thread_id);
3660
3661         /* Structs. */
3662         ALU(t, ip, ^);
3663
3664         /* Thread. */
3665         thread_ip_inc(p);
3666 }
3667
3668 static inline void
3669 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3670 {
3671         struct thread *t = &p->threads[p->thread_id];
3672         struct instruction *ip = t->ip;
3673
3674         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3675
3676         /* Structs. */
3677         ALU_MH(t, ip, ^);
3678
3679         /* Thread. */
3680         thread_ip_inc(p);
3681 }
3682
3683 static inline void
3684 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3685 {
3686         struct thread *t = &p->threads[p->thread_id];
3687         struct instruction *ip = t->ip;
3688
3689         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3690
3691         /* Structs. */
3692         ALU_HM_FAST(t, ip, ^);
3693
3694         /* Thread. */
3695         thread_ip_inc(p);
3696 }
3697
3698 static inline void
3699 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3700 {
3701         struct thread *t = &p->threads[p->thread_id];
3702         struct instruction *ip = t->ip;
3703
3704         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3705
3706         /* Structs. */
3707         ALU_HH_FAST(t, ip, ^);
3708
3709         /* Thread. */
3710         thread_ip_inc(p);
3711 }
3712
3713 static inline void
3714 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3715 {
3716         struct thread *t = &p->threads[p->thread_id];
3717         struct instruction *ip = t->ip;
3718
3719         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3720
3721         /* Structs. */
3722         ALU_I(t, ip, ^);
3723
3724         /* Thread. */
3725         thread_ip_inc(p);
3726 }
3727
3728 static inline void
3729 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3730 {
3731         struct thread *t = &p->threads[p->thread_id];
3732         struct instruction *ip = t->ip;
3733         uint8_t *dst_struct, *src_struct;
3734         uint16_t *dst16_ptr, dst;
3735         uint64_t *src64_ptr, src64, src64_mask, src;
3736         uint64_t r;
3737
3738         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3739
3740         /* Structs. */
3741         dst_struct = t->structs[ip->alu.dst.struct_id];
3742         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3743         dst = *dst16_ptr;
3744
3745         src_struct = t->structs[ip->alu.src.struct_id];
3746         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3747         src64 = *src64_ptr;
3748         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3749         src = src64 & src64_mask;
3750
3751         r = dst;
3752         r = ~r & 0xFFFF;
3753
3754         /* The first input (r) is a 16-bit number. The second and the third
3755          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3756          * three numbers (output r) is a 34-bit number.
3757          */
3758         r += (src >> 32) + (src & 0xFFFFFFFF);
3759
3760         /* The first input is a 16-bit number. The second input is an 18-bit
3761          * number. In the worst case scenario, the sum of the two numbers is a
3762          * 19-bit number.
3763          */
3764         r = (r & 0xFFFF) + (r >> 16);
3765
3766         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3767          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3768          */
3769         r = (r & 0xFFFF) + (r >> 16);
3770
3771         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3772          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3773          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3774          * therefore the output r is always a 16-bit number.
3775          */
3776         r = (r & 0xFFFF) + (r >> 16);
3777
3778         r = ~r & 0xFFFF;
3779         r = r ? r : 0xFFFF;
3780
3781         *dst16_ptr = (uint16_t)r;
3782
3783         /* Thread. */
3784         thread_ip_inc(p);
3785 }
3786
3787 static inline void
3788 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3789 {
3790         struct thread *t = &p->threads[p->thread_id];
3791         struct instruction *ip = t->ip;
3792         uint8_t *dst_struct, *src_struct;
3793         uint16_t *dst16_ptr, dst;
3794         uint64_t *src64_ptr, src64, src64_mask, src;
3795         uint64_t r;
3796
3797         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3798
3799         /* Structs. */
3800         dst_struct = t->structs[ip->alu.dst.struct_id];
3801         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3802         dst = *dst16_ptr;
3803
3804         src_struct = t->structs[ip->alu.src.struct_id];
3805         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3806         src64 = *src64_ptr;
3807         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3808         src = src64 & src64_mask;
3809
3810         r = dst;
3811         r = ~r & 0xFFFF;
3812
3813         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3814          * the following sequence of operations in 2's complement arithmetic:
3815          *    a '- b = (a - b) % 0xFFFF.
3816          *
3817          * In order to prevent an underflow for the below subtraction, in which
3818          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3819          * minuend), we first add a multiple of the 0xFFFF modulus to the
3820          * minuend. The number we add to the minuend needs to be a 34-bit number
3821          * or higher, so for readability reasons we picked the 36-bit multiple.
3822          * We are effectively turning the 16-bit minuend into a 36-bit number:
3823          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3824          */
3825         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3826
3827         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3828          * result (the output r) is a 36-bit number.
3829          */
3830         r -= (src >> 32) + (src & 0xFFFFFFFF);
3831
3832         /* The first input is a 16-bit number. The second input is a 20-bit
3833          * number. Their sum is a 21-bit number.
3834          */
3835         r = (r & 0xFFFF) + (r >> 16);
3836
3837         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3838          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3839          */
3840         r = (r & 0xFFFF) + (r >> 16);
3841
3842         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3843          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3844          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3845          * generated, therefore the output r is always a 16-bit number.
3846          */
3847         r = (r & 0xFFFF) + (r >> 16);
3848
3849         r = ~r & 0xFFFF;
3850         r = r ? r : 0xFFFF;
3851
3852         *dst16_ptr = (uint16_t)r;
3853
3854         /* Thread. */
3855         thread_ip_inc(p);
3856 }
3857
3858 static inline void
3859 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3860 {
3861         struct thread *t = &p->threads[p->thread_id];
3862         struct instruction *ip = t->ip;
3863         uint8_t *dst_struct, *src_struct;
3864         uint16_t *dst16_ptr;
3865         uint32_t *src32_ptr;
3866         uint64_t r0, r1;
3867
3868         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3869
3870         /* Structs. */
3871         dst_struct = t->structs[ip->alu.dst.struct_id];
3872         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3873
3874         src_struct = t->structs[ip->alu.src.struct_id];
3875         src32_ptr = (uint32_t *)&src_struct[0];
3876
3877         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3878         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3879         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3880         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3881         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3882
3883         /* The first input is a 16-bit number. The second input is a 19-bit
3884          * number. Their sum is a 20-bit number.
3885          */
3886         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3887
3888         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3889          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3890          */
3891         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3892
3893         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3894          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3895          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3896          * generated, therefore the output r is always a 16-bit number.
3897          */
3898         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3899
3900         r0 = ~r0 & 0xFFFF;
3901         r0 = r0 ? r0 : 0xFFFF;
3902
3903         *dst16_ptr = (uint16_t)r0;
3904
3905         /* Thread. */
3906         thread_ip_inc(p);
3907 }
3908
3909 static inline void
3910 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3911 {
3912         struct thread *t = &p->threads[p->thread_id];
3913         struct instruction *ip = t->ip;
3914         uint8_t *dst_struct, *src_struct;
3915         uint16_t *dst16_ptr;
3916         uint32_t *src32_ptr;
3917         uint64_t r = 0;
3918         uint32_t i;
3919
3920         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3921
3922         /* Structs. */
3923         dst_struct = t->structs[ip->alu.dst.struct_id];
3924         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3925
3926         src_struct = t->structs[ip->alu.src.struct_id];
3927         src32_ptr = (uint32_t *)&src_struct[0];
3928
3929         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3930          * Therefore, in the worst case scenario, a 35-bit number is added to a
3931          * 16-bit number (the input r), so the output r is 36-bit number.
3932          */
3933         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3934                 r += *src32_ptr;
3935
3936         /* The first input is a 16-bit number. The second input is a 20-bit
3937          * number. Their sum is a 21-bit number.
3938          */
3939         r = (r & 0xFFFF) + (r >> 16);
3940
3941         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3942          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3943          */
3944         r = (r & 0xFFFF) + (r >> 16);
3945
3946         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3947          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3948          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3949          * generated, therefore the output r is always a 16-bit number.
3950          */
3951         r = (r & 0xFFFF) + (r >> 16);
3952
3953         r = ~r & 0xFFFF;
3954         r = r ? r : 0xFFFF;
3955
3956         *dst16_ptr = (uint16_t)r;
3957
3958         /* Thread. */
3959         thread_ip_inc(p);
3960 }
3961
3962 /*
3963  * Register array.
3964  */
3965 static struct regarray *
3966 regarray_find(struct rte_swx_pipeline *p, const char *name);
3967
3968 static int
3969 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3970                       struct action *action,
3971                       char **tokens,
3972                       int n_tokens,
3973                       struct instruction *instr,
3974                       struct instruction_data *data __rte_unused)
3975 {
3976         char *regarray = tokens[1], *idx = tokens[2];
3977         struct regarray *r;
3978         struct field *fidx;
3979         uint32_t idx_struct_id, idx_val;
3980
3981         CHECK(n_tokens == 3, EINVAL);
3982
3983         r = regarray_find(p, regarray);
3984         CHECK(r, EINVAL);
3985
3986         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3987         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3988         if (fidx) {
3989                 CHECK(!fidx->var_size, EINVAL);
3990
3991                 instr->type = INSTR_REGPREFETCH_RM;
3992                 if (idx[0] == 'h')
3993                         instr->type = INSTR_REGPREFETCH_RH;
3994
3995                 instr->regarray.regarray_id = r->id;
3996                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3997                 instr->regarray.idx.n_bits = fidx->n_bits;
3998                 instr->regarray.idx.offset = fidx->offset / 8;
3999                 instr->regarray.dstsrc_val = 0; /* Unused. */
4000                 return 0;
4001         }
4002
4003         /* REGPREFETCH_RI. */
4004         idx_val = strtoul(idx, &idx, 0);
4005         CHECK(!idx[0], EINVAL);
4006
4007         instr->type = INSTR_REGPREFETCH_RI;
4008         instr->regarray.regarray_id = r->id;
4009         instr->regarray.idx_val = idx_val;
4010         instr->regarray.dstsrc_val = 0; /* Unused. */
4011         return 0;
4012 }
4013
4014 static int
4015 instr_regrd_translate(struct rte_swx_pipeline *p,
4016                       struct action *action,
4017                       char **tokens,
4018                       int n_tokens,
4019                       struct instruction *instr,
4020                       struct instruction_data *data __rte_unused)
4021 {
4022         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4023         struct regarray *r;
4024         struct field *fdst, *fidx;
4025         uint32_t dst_struct_id, idx_struct_id, idx_val;
4026
4027         CHECK(n_tokens == 4, EINVAL);
4028
4029         r = regarray_find(p, regarray);
4030         CHECK(r, EINVAL);
4031
4032         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4033         CHECK(fdst, EINVAL);
4034         CHECK(!fdst->var_size, EINVAL);
4035
4036         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4037         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4038         if (fidx) {
4039                 CHECK(!fidx->var_size, EINVAL);
4040
4041                 instr->type = INSTR_REGRD_MRM;
4042                 if (dst[0] == 'h' && idx[0] != 'h')
4043                         instr->type = INSTR_REGRD_HRM;
4044                 if (dst[0] != 'h' && idx[0] == 'h')
4045                         instr->type = INSTR_REGRD_MRH;
4046                 if (dst[0] == 'h' && idx[0] == 'h')
4047                         instr->type = INSTR_REGRD_HRH;
4048
4049                 instr->regarray.regarray_id = r->id;
4050                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4051                 instr->regarray.idx.n_bits = fidx->n_bits;
4052                 instr->regarray.idx.offset = fidx->offset / 8;
4053                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4054                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4055                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4056                 return 0;
4057         }
4058
4059         /* REGRD_MRI, REGRD_HRI. */
4060         idx_val = strtoul(idx, &idx, 0);
4061         CHECK(!idx[0], EINVAL);
4062
4063         instr->type = INSTR_REGRD_MRI;
4064         if (dst[0] == 'h')
4065                 instr->type = INSTR_REGRD_HRI;
4066
4067         instr->regarray.regarray_id = r->id;
4068         instr->regarray.idx_val = idx_val;
4069         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4070         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4071         instr->regarray.dstsrc.offset = fdst->offset / 8;
4072         return 0;
4073 }
4074
4075 static int
4076 instr_regwr_translate(struct rte_swx_pipeline *p,
4077                       struct action *action,
4078                       char **tokens,
4079                       int n_tokens,
4080                       struct instruction *instr,
4081                       struct instruction_data *data __rte_unused)
4082 {
4083         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4084         struct regarray *r;
4085         struct field *fidx, *fsrc;
4086         uint64_t src_val;
4087         uint32_t idx_struct_id, idx_val, src_struct_id;
4088
4089         CHECK(n_tokens == 4, EINVAL);
4090
4091         r = regarray_find(p, regarray);
4092         CHECK(r, EINVAL);
4093
4094         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4095         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4096         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4097         if (fidx && fsrc) {
4098                 CHECK(!fidx->var_size, EINVAL);
4099                 CHECK(!fsrc->var_size, EINVAL);
4100
4101                 instr->type = INSTR_REGWR_RMM;
4102                 if (idx[0] == 'h' && src[0] != 'h')
4103                         instr->type = INSTR_REGWR_RHM;
4104                 if (idx[0] != 'h' && src[0] == 'h')
4105                         instr->type = INSTR_REGWR_RMH;
4106                 if (idx[0] == 'h' && src[0] == 'h')
4107                         instr->type = INSTR_REGWR_RHH;
4108
4109                 instr->regarray.regarray_id = r->id;
4110                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4111                 instr->regarray.idx.n_bits = fidx->n_bits;
4112                 instr->regarray.idx.offset = fidx->offset / 8;
4113                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4114                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4115                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4116                 return 0;
4117         }
4118
4119         /* REGWR_RHI, REGWR_RMI. */
4120         if (fidx && !fsrc) {
4121                 CHECK(!fidx->var_size, EINVAL);
4122
4123                 src_val = strtoull(src, &src, 0);
4124                 CHECK(!src[0], EINVAL);
4125
4126                 instr->type = INSTR_REGWR_RMI;
4127                 if (idx[0] == 'h')
4128                         instr->type = INSTR_REGWR_RHI;
4129
4130                 instr->regarray.regarray_id = r->id;
4131                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4132                 instr->regarray.idx.n_bits = fidx->n_bits;
4133                 instr->regarray.idx.offset = fidx->offset / 8;
4134                 instr->regarray.dstsrc_val = src_val;
4135                 return 0;
4136         }
4137
4138         /* REGWR_RIH, REGWR_RIM. */
4139         if (!fidx && fsrc) {
4140                 idx_val = strtoul(idx, &idx, 0);
4141                 CHECK(!idx[0], EINVAL);
4142
4143                 CHECK(!fsrc->var_size, EINVAL);
4144
4145                 instr->type = INSTR_REGWR_RIM;
4146                 if (src[0] == 'h')
4147                         instr->type = INSTR_REGWR_RIH;
4148
4149                 instr->regarray.regarray_id = r->id;
4150                 instr->regarray.idx_val = idx_val;
4151                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4152                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4153                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4154                 return 0;
4155         }
4156
4157         /* REGWR_RII. */
4158         src_val = strtoull(src, &src, 0);
4159         CHECK(!src[0], EINVAL);
4160
4161         idx_val = strtoul(idx, &idx, 0);
4162         CHECK(!idx[0], EINVAL);
4163
4164         instr->type = INSTR_REGWR_RII;
4165         instr->regarray.idx_val = idx_val;
4166         instr->regarray.dstsrc_val = src_val;
4167
4168         return 0;
4169 }
4170
4171 static int
4172 instr_regadd_translate(struct rte_swx_pipeline *p,
4173                        struct action *action,
4174                        char **tokens,
4175                        int n_tokens,
4176                        struct instruction *instr,
4177                        struct instruction_data *data __rte_unused)
4178 {
4179         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4180         struct regarray *r;
4181         struct field *fidx, *fsrc;
4182         uint64_t src_val;
4183         uint32_t idx_struct_id, idx_val, src_struct_id;
4184
4185         CHECK(n_tokens == 4, EINVAL);
4186
4187         r = regarray_find(p, regarray);
4188         CHECK(r, EINVAL);
4189
4190         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4191         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4192         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4193         if (fidx && fsrc) {
4194                 CHECK(!fidx->var_size, EINVAL);
4195                 CHECK(!fsrc->var_size, EINVAL);
4196
4197                 instr->type = INSTR_REGADD_RMM;
4198                 if (idx[0] == 'h' && src[0] != 'h')
4199                         instr->type = INSTR_REGADD_RHM;
4200                 if (idx[0] != 'h' && src[0] == 'h')
4201                         instr->type = INSTR_REGADD_RMH;
4202                 if (idx[0] == 'h' && src[0] == 'h')
4203                         instr->type = INSTR_REGADD_RHH;
4204
4205                 instr->regarray.regarray_id = r->id;
4206                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4207                 instr->regarray.idx.n_bits = fidx->n_bits;
4208                 instr->regarray.idx.offset = fidx->offset / 8;
4209                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4210                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4211                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4212                 return 0;
4213         }
4214
4215         /* REGADD_RHI, REGADD_RMI. */
4216         if (fidx && !fsrc) {
4217                 CHECK(!fidx->var_size, EINVAL);
4218
4219                 src_val = strtoull(src, &src, 0);
4220                 CHECK(!src[0], EINVAL);
4221
4222                 instr->type = INSTR_REGADD_RMI;
4223                 if (idx[0] == 'h')
4224                         instr->type = INSTR_REGADD_RHI;
4225
4226                 instr->regarray.regarray_id = r->id;
4227                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4228                 instr->regarray.idx.n_bits = fidx->n_bits;
4229                 instr->regarray.idx.offset = fidx->offset / 8;
4230                 instr->regarray.dstsrc_val = src_val;
4231                 return 0;
4232         }
4233
4234         /* REGADD_RIH, REGADD_RIM. */
4235         if (!fidx && fsrc) {
4236                 idx_val = strtoul(idx, &idx, 0);
4237                 CHECK(!idx[0], EINVAL);
4238
4239                 CHECK(!fsrc->var_size, EINVAL);
4240
4241                 instr->type = INSTR_REGADD_RIM;
4242                 if (src[0] == 'h')
4243                         instr->type = INSTR_REGADD_RIH;
4244
4245                 instr->regarray.regarray_id = r->id;
4246                 instr->regarray.idx_val = idx_val;
4247                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4248                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4249                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4250                 return 0;
4251         }
4252
4253         /* REGADD_RII. */
4254         src_val = strtoull(src, &src, 0);
4255         CHECK(!src[0], EINVAL);
4256
4257         idx_val = strtoul(idx, &idx, 0);
4258         CHECK(!idx[0], EINVAL);
4259
4260         instr->type = INSTR_REGADD_RII;
4261         instr->regarray.idx_val = idx_val;
4262         instr->regarray.dstsrc_val = src_val;
4263         return 0;
4264 }
4265
4266 static inline uint64_t *
4267 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
4268 {
4269         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4270         return r->regarray;
4271 }
4272
4273 static inline uint64_t
4274 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4275 {
4276         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4277
4278         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4279         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4280         uint64_t idx64 = *idx64_ptr;
4281         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
4282         uint64_t idx = idx64 & idx64_mask & r->size_mask;
4283
4284         return idx;
4285 }
4286
4287 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4288
4289 static inline uint64_t
4290 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
4291 {
4292         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4293
4294         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
4295         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
4296         uint64_t idx64 = *idx64_ptr;
4297         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
4298
4299         return idx;
4300 }
4301
4302 #else
4303
4304 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
4305
4306 #endif
4307
4308 static inline uint64_t
4309 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
4310 {
4311         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
4312
4313         uint64_t idx = ip->regarray.idx_val & r->size_mask;
4314
4315         return idx;
4316 }
4317
4318 static inline uint64_t
4319 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
4320 {
4321         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4322         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4323         uint64_t src64 = *src64_ptr;
4324         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4325         uint64_t src = src64 & src64_mask;
4326
4327         return src;
4328 }
4329
4330 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4331
4332 static inline uint64_t
4333 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
4334 {
4335         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
4336         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
4337         uint64_t src64 = *src64_ptr;
4338         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
4339
4340         return src;
4341 }
4342
4343 #else
4344
4345 #define instr_regarray_src_nbo instr_regarray_src_hbo
4346
4347 #endif
4348
4349 static inline void
4350 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4351 {
4352         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4353         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4354         uint64_t dst64 = *dst64_ptr;
4355         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4356
4357         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4358
4359 }
4360
4361 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4362
4363 static inline void
4364 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
4365 {
4366         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
4367         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
4368         uint64_t dst64 = *dst64_ptr;
4369         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
4370
4371         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
4372         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4373 }
4374
4375 #else
4376
4377 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
4378
4379 #endif
4380
4381 static inline void
4382 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4383 {
4384         struct thread *t = &p->threads[p->thread_id];
4385         struct instruction *ip = t->ip;
4386         uint64_t *regarray, idx;
4387
4388         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
4389
4390         /* Structs. */
4391         regarray = instr_regarray_regarray(p, ip);
4392         idx = instr_regarray_idx_nbo(p, t, ip);
4393         rte_prefetch0(&regarray[idx]);
4394
4395         /* Thread. */
4396         thread_ip_inc(p);
4397 }
4398
4399 static inline void
4400 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4401 {
4402         struct thread *t = &p->threads[p->thread_id];
4403         struct instruction *ip = t->ip;
4404         uint64_t *regarray, idx;
4405
4406         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
4407
4408         /* Structs. */
4409         regarray = instr_regarray_regarray(p, ip);
4410         idx = instr_regarray_idx_hbo(p, t, ip);
4411         rte_prefetch0(&regarray[idx]);
4412
4413         /* Thread. */
4414         thread_ip_inc(p);
4415 }
4416
4417 static inline void
4418 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4419 {
4420         struct thread *t = &p->threads[p->thread_id];
4421         struct instruction *ip = t->ip;
4422         uint64_t *regarray, idx;
4423
4424         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
4425
4426         /* Structs. */
4427         regarray = instr_regarray_regarray(p, ip);
4428         idx = instr_regarray_idx_imm(p, ip);
4429         rte_prefetch0(&regarray[idx]);
4430
4431         /* Thread. */
4432         thread_ip_inc(p);
4433 }
4434
4435 static inline void
4436 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4437 {
4438         struct thread *t = &p->threads[p->thread_id];
4439         struct instruction *ip = t->ip;
4440         uint64_t *regarray, idx;
4441
4442         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
4443
4444         /* Structs. */
4445         regarray = instr_regarray_regarray(p, ip);
4446         idx = instr_regarray_idx_nbo(p, t, ip);
4447         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4448
4449         /* Thread. */
4450         thread_ip_inc(p);
4451 }
4452
4453 static inline void
4454 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4455 {
4456         struct thread *t = &p->threads[p->thread_id];
4457         struct instruction *ip = t->ip;
4458         uint64_t *regarray, idx;
4459
4460         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
4461
4462         /* Structs. */
4463         regarray = instr_regarray_regarray(p, ip);
4464         idx = instr_regarray_idx_hbo(p, t, ip);
4465         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4466
4467         /* Thread. */
4468         thread_ip_inc(p);
4469 }
4470
4471 static inline void
4472 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4473 {
4474         struct thread *t = &p->threads[p->thread_id];
4475         struct instruction *ip = t->ip;
4476         uint64_t *regarray, idx;
4477
4478         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
4479
4480         /* Structs. */
4481         regarray = instr_regarray_regarray(p, ip);
4482         idx = instr_regarray_idx_nbo(p, t, ip);
4483         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4484
4485         /* Thread. */
4486         thread_ip_inc(p);
4487 }
4488
4489 static inline void
4490 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4491 {
4492         struct thread *t = &p->threads[p->thread_id];
4493         struct instruction *ip = t->ip;
4494         uint64_t *regarray, idx;
4495
4496         /* Structs. */
4497         regarray = instr_regarray_regarray(p, ip);
4498         idx = instr_regarray_idx_hbo(p, t, ip);
4499         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4500
4501         /* Thread. */
4502         thread_ip_inc(p);
4503 }
4504
4505 static inline void
4506 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4507 {
4508         struct thread *t = &p->threads[p->thread_id];
4509         struct instruction *ip = t->ip;
4510         uint64_t *regarray, idx;
4511
4512         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
4513
4514         /* Structs. */
4515         regarray = instr_regarray_regarray(p, ip);
4516         idx = instr_regarray_idx_imm(p, ip);
4517         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
4518
4519         /* Thread. */
4520         thread_ip_inc(p);
4521 }
4522
4523 static inline void
4524 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4525 {
4526         struct thread *t = &p->threads[p->thread_id];
4527         struct instruction *ip = t->ip;
4528         uint64_t *regarray, idx;
4529
4530         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
4531
4532         /* Structs. */
4533         regarray = instr_regarray_regarray(p, ip);
4534         idx = instr_regarray_idx_imm(p, ip);
4535         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
4536
4537         /* Thread. */
4538         thread_ip_inc(p);
4539 }
4540
4541 static inline void
4542 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4543 {
4544         struct thread *t = &p->threads[p->thread_id];
4545         struct instruction *ip = t->ip;
4546         uint64_t *regarray, idx, src;
4547
4548         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
4549
4550         /* Structs. */
4551         regarray = instr_regarray_regarray(p, ip);
4552         idx = instr_regarray_idx_nbo(p, t, ip);
4553         src = instr_regarray_src_nbo(t, ip);
4554         regarray[idx] = src;
4555
4556         /* Thread. */
4557         thread_ip_inc(p);
4558 }
4559
4560 static inline void
4561 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4562 {
4563         struct thread *t = &p->threads[p->thread_id];
4564         struct instruction *ip = t->ip;
4565         uint64_t *regarray, idx, src;
4566
4567         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
4568
4569         /* Structs. */
4570         regarray = instr_regarray_regarray(p, ip);
4571         idx = instr_regarray_idx_nbo(p, t, ip);
4572         src = instr_regarray_src_hbo(t, ip);
4573         regarray[idx] = src;
4574
4575         /* Thread. */
4576         thread_ip_inc(p);
4577 }
4578
4579 static inline void
4580 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4581 {
4582         struct thread *t = &p->threads[p->thread_id];
4583         struct instruction *ip = t->ip;
4584         uint64_t *regarray, idx, src;
4585
4586         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
4587
4588         /* Structs. */
4589         regarray = instr_regarray_regarray(p, ip);
4590         idx = instr_regarray_idx_hbo(p, t, ip);
4591         src = instr_regarray_src_nbo(t, ip);
4592         regarray[idx] = src;
4593
4594         /* Thread. */
4595         thread_ip_inc(p);
4596 }
4597
4598 static inline void
4599 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4600 {
4601         struct thread *t = &p->threads[p->thread_id];
4602         struct instruction *ip = t->ip;
4603         uint64_t *regarray, idx, src;
4604
4605         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
4606
4607         /* Structs. */
4608         regarray = instr_regarray_regarray(p, ip);
4609         idx = instr_regarray_idx_hbo(p, t, ip);
4610         src = instr_regarray_src_hbo(t, ip);
4611         regarray[idx] = src;
4612
4613         /* Thread. */
4614         thread_ip_inc(p);
4615 }
4616
4617 static inline void
4618 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4619 {
4620         struct thread *t = &p->threads[p->thread_id];
4621         struct instruction *ip = t->ip;
4622         uint64_t *regarray, idx, src;
4623
4624         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
4625
4626         /* Structs. */
4627         regarray = instr_regarray_regarray(p, ip);
4628         idx = instr_regarray_idx_nbo(p, t, ip);
4629         src = ip->regarray.dstsrc_val;
4630         regarray[idx] = src;
4631
4632         /* Thread. */
4633         thread_ip_inc(p);
4634 }
4635
4636 static inline void
4637 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4638 {
4639         struct thread *t = &p->threads[p->thread_id];
4640         struct instruction *ip = t->ip;
4641         uint64_t *regarray, idx, src;
4642
4643         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
4644
4645         /* Structs. */
4646         regarray = instr_regarray_regarray(p, ip);
4647         idx = instr_regarray_idx_hbo(p, t, ip);
4648         src = ip->regarray.dstsrc_val;
4649         regarray[idx] = src;
4650
4651         /* Thread. */
4652         thread_ip_inc(p);
4653 }
4654
4655 static inline void
4656 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4657 {
4658         struct thread *t = &p->threads[p->thread_id];
4659         struct instruction *ip = t->ip;
4660         uint64_t *regarray, idx, src;
4661
4662         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
4663
4664         /* Structs. */
4665         regarray = instr_regarray_regarray(p, ip);
4666         idx = instr_regarray_idx_imm(p, ip);
4667         src = instr_regarray_src_nbo(t, ip);
4668         regarray[idx] = src;
4669
4670         /* Thread. */
4671         thread_ip_inc(p);
4672 }
4673
4674 static inline void
4675 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4676 {
4677         struct thread *t = &p->threads[p->thread_id];
4678         struct instruction *ip = t->ip;
4679         uint64_t *regarray, idx, src;
4680
4681         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
4682
4683         /* Structs. */
4684         regarray = instr_regarray_regarray(p, ip);
4685         idx = instr_regarray_idx_imm(p, ip);
4686         src = instr_regarray_src_hbo(t, ip);
4687         regarray[idx] = src;
4688
4689         /* Thread. */
4690         thread_ip_inc(p);
4691 }
4692
4693 static inline void
4694 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4695 {
4696         struct thread *t = &p->threads[p->thread_id];
4697         struct instruction *ip = t->ip;
4698         uint64_t *regarray, idx, src;
4699
4700         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
4701
4702         /* Structs. */
4703         regarray = instr_regarray_regarray(p, ip);
4704         idx = instr_regarray_idx_imm(p, ip);
4705         src = ip->regarray.dstsrc_val;
4706         regarray[idx] = src;
4707
4708         /* Thread. */
4709         thread_ip_inc(p);
4710 }
4711
4712 static inline void
4713 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4714 {
4715         struct thread *t = &p->threads[p->thread_id];
4716         struct instruction *ip = t->ip;
4717         uint64_t *regarray, idx, src;
4718
4719         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
4720
4721         /* Structs. */
4722         regarray = instr_regarray_regarray(p, ip);
4723         idx = instr_regarray_idx_nbo(p, t, ip);
4724         src = instr_regarray_src_nbo(t, ip);
4725         regarray[idx] += src;
4726
4727         /* Thread. */
4728         thread_ip_inc(p);
4729 }
4730
4731 static inline void
4732 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4733 {
4734         struct thread *t = &p->threads[p->thread_id];
4735         struct instruction *ip = t->ip;
4736         uint64_t *regarray, idx, src;
4737
4738         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
4739
4740         /* Structs. */
4741         regarray = instr_regarray_regarray(p, ip);
4742         idx = instr_regarray_idx_nbo(p, t, ip);
4743         src = instr_regarray_src_hbo(t, ip);
4744         regarray[idx] += src;
4745
4746         /* Thread. */
4747         thread_ip_inc(p);
4748 }
4749
4750 static inline void
4751 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4752 {
4753         struct thread *t = &p->threads[p->thread_id];
4754         struct instruction *ip = t->ip;
4755         uint64_t *regarray, idx, src;
4756
4757         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
4758
4759         /* Structs. */
4760         regarray = instr_regarray_regarray(p, ip);
4761         idx = instr_regarray_idx_hbo(p, t, ip);
4762         src = instr_regarray_src_nbo(t, ip);
4763         regarray[idx] += src;
4764
4765         /* Thread. */
4766         thread_ip_inc(p);
4767 }
4768
4769 static inline void
4770 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4771 {
4772         struct thread *t = &p->threads[p->thread_id];
4773         struct instruction *ip = t->ip;
4774         uint64_t *regarray, idx, src;
4775
4776         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
4777
4778         /* Structs. */
4779         regarray = instr_regarray_regarray(p, ip);
4780         idx = instr_regarray_idx_hbo(p, t, ip);
4781         src = instr_regarray_src_hbo(t, ip);
4782         regarray[idx] += src;
4783
4784         /* Thread. */
4785         thread_ip_inc(p);
4786 }
4787
4788 static inline void
4789 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4790 {
4791         struct thread *t = &p->threads[p->thread_id];
4792         struct instruction *ip = t->ip;
4793         uint64_t *regarray, idx, src;
4794
4795         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
4796
4797         /* Structs. */
4798         regarray = instr_regarray_regarray(p, ip);
4799         idx = instr_regarray_idx_nbo(p, t, ip);
4800         src = ip->regarray.dstsrc_val;
4801         regarray[idx] += src;
4802
4803         /* Thread. */
4804         thread_ip_inc(p);
4805 }
4806
4807 static inline void
4808 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4809 {
4810         struct thread *t = &p->threads[p->thread_id];
4811         struct instruction *ip = t->ip;
4812         uint64_t *regarray, idx, src;
4813
4814         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
4815
4816         /* Structs. */
4817         regarray = instr_regarray_regarray(p, ip);
4818         idx = instr_regarray_idx_hbo(p, t, ip);
4819         src = ip->regarray.dstsrc_val;
4820         regarray[idx] += src;
4821
4822         /* Thread. */
4823         thread_ip_inc(p);
4824 }
4825
4826 static inline void
4827 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4828 {
4829         struct thread *t = &p->threads[p->thread_id];
4830         struct instruction *ip = t->ip;
4831         uint64_t *regarray, idx, src;
4832
4833         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
4834
4835         /* Structs. */
4836         regarray = instr_regarray_regarray(p, ip);
4837         idx = instr_regarray_idx_imm(p, ip);
4838         src = instr_regarray_src_nbo(t, ip);
4839         regarray[idx] += src;
4840
4841         /* Thread. */
4842         thread_ip_inc(p);
4843 }
4844
4845 static inline void
4846 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4847 {
4848         struct thread *t = &p->threads[p->thread_id];
4849         struct instruction *ip = t->ip;
4850         uint64_t *regarray, idx, src;
4851
4852         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4853
4854         /* Structs. */
4855         regarray = instr_regarray_regarray(p, ip);
4856         idx = instr_regarray_idx_imm(p, ip);
4857         src = instr_regarray_src_hbo(t, ip);
4858         regarray[idx] += src;
4859
4860         /* Thread. */
4861         thread_ip_inc(p);
4862 }
4863
4864 static inline void
4865 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4866 {
4867         struct thread *t = &p->threads[p->thread_id];
4868         struct instruction *ip = t->ip;
4869         uint64_t *regarray, idx, src;
4870
4871         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4872
4873         /* Structs. */
4874         regarray = instr_regarray_regarray(p, ip);
4875         idx = instr_regarray_idx_imm(p, ip);
4876         src = ip->regarray.dstsrc_val;
4877         regarray[idx] += src;
4878
4879         /* Thread. */
4880         thread_ip_inc(p);
4881 }
4882
4883 /*
4884  * metarray.
4885  */
4886 static struct metarray *
4887 metarray_find(struct rte_swx_pipeline *p, const char *name);
4888
4889 static int
4890 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4891                             struct action *action,
4892                             char **tokens,
4893                             int n_tokens,
4894                             struct instruction *instr,
4895                             struct instruction_data *data __rte_unused)
4896 {
4897         char *metarray = tokens[1], *idx = tokens[2];
4898         struct metarray *m;
4899         struct field *fidx;
4900         uint32_t idx_struct_id, idx_val;
4901
4902         CHECK(n_tokens == 3, EINVAL);
4903
4904         m = metarray_find(p, metarray);
4905         CHECK(m, EINVAL);
4906
4907         /* METPREFETCH_H, METPREFETCH_M. */
4908         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4909         if (fidx) {
4910                 CHECK(!fidx->var_size, EINVAL);
4911
4912                 instr->type = INSTR_METPREFETCH_M;
4913                 if (idx[0] == 'h')
4914                         instr->type = INSTR_METPREFETCH_H;
4915
4916                 instr->meter.metarray_id = m->id;
4917                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4918                 instr->meter.idx.n_bits = fidx->n_bits;
4919                 instr->meter.idx.offset = fidx->offset / 8;
4920                 return 0;
4921         }
4922
4923         /* METPREFETCH_I. */
4924         idx_val = strtoul(idx, &idx, 0);
4925         CHECK(!idx[0], EINVAL);
4926
4927         instr->type = INSTR_METPREFETCH_I;
4928         instr->meter.metarray_id = m->id;
4929         instr->meter.idx_val = idx_val;
4930         return 0;
4931 }
4932
4933 static int
4934 instr_meter_translate(struct rte_swx_pipeline *p,
4935                       struct action *action,
4936                       char **tokens,
4937                       int n_tokens,
4938                       struct instruction *instr,
4939                       struct instruction_data *data __rte_unused)
4940 {
4941         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4942         char *color_in = tokens[4], *color_out = tokens[5];
4943         struct metarray *m;
4944         struct field *fidx, *flength, *fcin, *fcout;
4945         uint32_t idx_struct_id, length_struct_id;
4946         uint32_t color_in_struct_id, color_out_struct_id;
4947
4948         CHECK(n_tokens == 6, EINVAL);
4949
4950         m = metarray_find(p, metarray);
4951         CHECK(m, EINVAL);
4952
4953         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4954
4955         flength = struct_field_parse(p, action, length, &length_struct_id);
4956         CHECK(flength, EINVAL);
4957         CHECK(!flength->var_size, EINVAL);
4958
4959         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4960
4961         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4962         CHECK(fcout, EINVAL);
4963         CHECK(!fcout->var_size, EINVAL);
4964
4965         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4966         if (fidx && fcin) {
4967                 CHECK(!fidx->var_size, EINVAL);
4968                 CHECK(!fcin->var_size, EINVAL);
4969
4970                 instr->type = INSTR_METER_MMM;
4971                 if (idx[0] == 'h' && length[0] == 'h')
4972                         instr->type = INSTR_METER_HHM;
4973                 if (idx[0] == 'h' && length[0] != 'h')
4974                         instr->type = INSTR_METER_HMM;
4975                 if (idx[0] != 'h' && length[0] == 'h')
4976                         instr->type = INSTR_METER_MHM;
4977
4978                 instr->meter.metarray_id = m->id;
4979
4980                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4981                 instr->meter.idx.n_bits = fidx->n_bits;
4982                 instr->meter.idx.offset = fidx->offset / 8;
4983
4984                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4985                 instr->meter.length.n_bits = flength->n_bits;
4986                 instr->meter.length.offset = flength->offset / 8;
4987
4988                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4989                 instr->meter.color_in.n_bits = fcin->n_bits;
4990                 instr->meter.color_in.offset = fcin->offset / 8;
4991
4992                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4993                 instr->meter.color_out.n_bits = fcout->n_bits;
4994                 instr->meter.color_out.offset = fcout->offset / 8;
4995
4996                 return 0;
4997         }
4998
4999         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5000         if (fidx && !fcin) {
5001                 uint32_t color_in_val;
5002
5003                 CHECK(!fidx->var_size, EINVAL);
5004
5005                 color_in_val = strtoul(color_in, &color_in, 0);
5006                 CHECK(!color_in[0], EINVAL);
5007
5008                 instr->type = INSTR_METER_MMI;
5009                 if (idx[0] == 'h' && length[0] == 'h')
5010                         instr->type = INSTR_METER_HHI;
5011                 if (idx[0] == 'h' && length[0] != 'h')
5012                         instr->type = INSTR_METER_HMI;
5013                 if (idx[0] != 'h' && length[0] == 'h')
5014                         instr->type = INSTR_METER_MHI;
5015
5016                 instr->meter.metarray_id = m->id;
5017
5018                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5019                 instr->meter.idx.n_bits = fidx->n_bits;
5020                 instr->meter.idx.offset = fidx->offset / 8;
5021
5022                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5023                 instr->meter.length.n_bits = flength->n_bits;
5024                 instr->meter.length.offset = flength->offset / 8;
5025
5026                 instr->meter.color_in_val = color_in_val;
5027
5028                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5029                 instr->meter.color_out.n_bits = fcout->n_bits;
5030                 instr->meter.color_out.offset = fcout->offset / 8;
5031
5032                 return 0;
5033         }
5034
5035         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5036         if (!fidx && fcin) {
5037                 uint32_t idx_val;
5038
5039                 idx_val = strtoul(idx, &idx, 0);
5040                 CHECK(!idx[0], EINVAL);
5041
5042                 CHECK(!fcin->var_size, EINVAL);
5043
5044                 instr->type = INSTR_METER_IMM;
5045                 if (length[0] == 'h')
5046                         instr->type = INSTR_METER_IHM;
5047
5048                 instr->meter.metarray_id = m->id;
5049
5050                 instr->meter.idx_val = idx_val;
5051
5052                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5053                 instr->meter.length.n_bits = flength->n_bits;
5054                 instr->meter.length.offset = flength->offset / 8;
5055
5056                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5057                 instr->meter.color_in.n_bits = fcin->n_bits;
5058                 instr->meter.color_in.offset = fcin->offset / 8;
5059
5060                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5061                 instr->meter.color_out.n_bits = fcout->n_bits;
5062                 instr->meter.color_out.offset = fcout->offset / 8;
5063
5064                 return 0;
5065         }
5066
5067         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5068         if (!fidx && !fcin) {
5069                 uint32_t idx_val, color_in_val;
5070
5071                 idx_val = strtoul(idx, &idx, 0);
5072                 CHECK(!idx[0], EINVAL);
5073
5074                 color_in_val = strtoul(color_in, &color_in, 0);
5075                 CHECK(!color_in[0], EINVAL);
5076
5077                 instr->type = INSTR_METER_IMI;
5078                 if (length[0] == 'h')
5079                         instr->type = INSTR_METER_IHI;
5080
5081                 instr->meter.metarray_id = m->id;
5082
5083                 instr->meter.idx_val = idx_val;
5084
5085                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5086                 instr->meter.length.n_bits = flength->n_bits;
5087                 instr->meter.length.offset = flength->offset / 8;
5088
5089                 instr->meter.color_in_val = color_in_val;
5090
5091                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5092                 instr->meter.color_out.n_bits = fcout->n_bits;
5093                 instr->meter.color_out.offset = fcout->offset / 8;
5094
5095                 return 0;
5096         }
5097
5098         CHECK(0, EINVAL);
5099 }
5100
5101 static inline struct meter *
5102 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5103 {
5104         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5105
5106         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5107         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5108         uint64_t idx64 = *idx64_ptr;
5109         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5110         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5111
5112         return &r->metarray[idx];
5113 }
5114
5115 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5116
5117 static inline struct meter *
5118 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5119 {
5120         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5121
5122         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5123         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5124         uint64_t idx64 = *idx64_ptr;
5125         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5126
5127         return &r->metarray[idx];
5128 }
5129
5130 #else
5131
5132 #define instr_meter_idx_nbo instr_meter_idx_hbo
5133
5134 #endif
5135
5136 static inline struct meter *
5137 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5138 {
5139         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5140
5141         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5142
5143         return &r->metarray[idx];
5144 }
5145
5146 static inline uint32_t
5147 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5148 {
5149         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5150         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5151         uint64_t src64 = *src64_ptr;
5152         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
5153         uint64_t src = src64 & src64_mask;
5154
5155         return (uint32_t)src;
5156 }
5157
5158 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5159
5160 static inline uint32_t
5161 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
5162 {
5163         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
5164         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
5165         uint64_t src64 = *src64_ptr;
5166         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
5167
5168         return (uint32_t)src;
5169 }
5170
5171 #else
5172
5173 #define instr_meter_length_nbo instr_meter_length_hbo
5174
5175 #endif
5176
5177 static inline enum rte_color
5178 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
5179 {
5180         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
5181         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
5182         uint64_t src64 = *src64_ptr;
5183         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
5184         uint64_t src = src64 & src64_mask;
5185
5186         return (enum rte_color)src;
5187 }
5188
5189 static inline void
5190 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
5191 {
5192         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
5193         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
5194         uint64_t dst64 = *dst64_ptr;
5195         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
5196
5197         uint64_t src = (uint64_t)color_out;
5198
5199         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5200 }
5201
5202 static inline void
5203 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5204 {
5205         struct thread *t = &p->threads[p->thread_id];
5206         struct instruction *ip = t->ip;
5207         struct meter *m;
5208
5209         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
5210
5211         /* Structs. */
5212         m = instr_meter_idx_nbo(p, t, ip);
5213         rte_prefetch0(m);
5214
5215         /* Thread. */
5216         thread_ip_inc(p);
5217 }
5218
5219 static inline void
5220 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5221 {
5222         struct thread *t = &p->threads[p->thread_id];
5223         struct instruction *ip = t->ip;
5224         struct meter *m;
5225
5226         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
5227
5228         /* Structs. */
5229         m = instr_meter_idx_hbo(p, t, ip);
5230         rte_prefetch0(m);
5231
5232         /* Thread. */
5233         thread_ip_inc(p);
5234 }
5235
5236 static inline void
5237 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5238 {
5239         struct thread *t = &p->threads[p->thread_id];
5240         struct instruction *ip = t->ip;
5241         struct meter *m;
5242
5243         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
5244
5245         /* Structs. */
5246         m = instr_meter_idx_imm(p, ip);
5247         rte_prefetch0(m);
5248
5249         /* Thread. */
5250         thread_ip_inc(p);
5251 }
5252
5253 static inline void
5254 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5255 {
5256         struct thread *t = &p->threads[p->thread_id];
5257         struct instruction *ip = t->ip;
5258         struct meter *m;
5259         uint64_t time, n_pkts, n_bytes;
5260         uint32_t length;
5261         enum rte_color color_in, color_out;
5262
5263         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
5264
5265         /* Structs. */
5266         m = instr_meter_idx_nbo(p, t, ip);
5267         rte_prefetch0(m->n_pkts);
5268         time = rte_get_tsc_cycles();
5269         length = instr_meter_length_nbo(t, ip);
5270         color_in = instr_meter_color_in_hbo(t, ip);
5271
5272         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5273                 &m->profile->profile,
5274                 time,
5275                 length,
5276                 color_in);
5277
5278         color_out &= m->color_mask;
5279
5280         n_pkts = m->n_pkts[color_out];
5281         n_bytes = m->n_bytes[color_out];
5282
5283         instr_meter_color_out_hbo_set(t, ip, color_out);
5284
5285         m->n_pkts[color_out] = n_pkts + 1;
5286         m->n_bytes[color_out] = n_bytes + length;
5287
5288         /* Thread. */
5289         thread_ip_inc(p);
5290 }
5291
5292 static inline void
5293 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5294 {
5295         struct thread *t = &p->threads[p->thread_id];
5296         struct instruction *ip = t->ip;
5297         struct meter *m;
5298         uint64_t time, n_pkts, n_bytes;
5299         uint32_t length;
5300         enum rte_color color_in, color_out;
5301
5302         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
5303
5304         /* Structs. */
5305         m = instr_meter_idx_nbo(p, t, ip);
5306         rte_prefetch0(m->n_pkts);
5307         time = rte_get_tsc_cycles();
5308         length = instr_meter_length_nbo(t, ip);
5309         color_in = (enum rte_color)ip->meter.color_in_val;
5310
5311         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5312                 &m->profile->profile,
5313                 time,
5314                 length,
5315                 color_in);
5316
5317         color_out &= m->color_mask;
5318
5319         n_pkts = m->n_pkts[color_out];
5320         n_bytes = m->n_bytes[color_out];
5321
5322         instr_meter_color_out_hbo_set(t, ip, color_out);
5323
5324         m->n_pkts[color_out] = n_pkts + 1;
5325         m->n_bytes[color_out] = n_bytes + length;
5326
5327         /* Thread. */
5328         thread_ip_inc(p);
5329 }
5330
5331 static inline void
5332 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5333 {
5334         struct thread *t = &p->threads[p->thread_id];
5335         struct instruction *ip = t->ip;
5336         struct meter *m;
5337         uint64_t time, n_pkts, n_bytes;
5338         uint32_t length;
5339         enum rte_color color_in, color_out;
5340
5341         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
5342
5343         /* Structs. */
5344         m = instr_meter_idx_nbo(p, t, ip);
5345         rte_prefetch0(m->n_pkts);
5346         time = rte_get_tsc_cycles();
5347         length = instr_meter_length_hbo(t, ip);
5348         color_in = instr_meter_color_in_hbo(t, ip);
5349
5350         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5351                 &m->profile->profile,
5352                 time,
5353                 length,
5354                 color_in);
5355
5356         color_out &= m->color_mask;
5357
5358         n_pkts = m->n_pkts[color_out];
5359         n_bytes = m->n_bytes[color_out];
5360
5361         instr_meter_color_out_hbo_set(t, ip, color_out);
5362
5363         m->n_pkts[color_out] = n_pkts + 1;
5364         m->n_bytes[color_out] = n_bytes + length;
5365
5366         /* Thread. */
5367         thread_ip_inc(p);
5368 }
5369 static inline void
5370 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5371 {
5372         struct thread *t = &p->threads[p->thread_id];
5373         struct instruction *ip = t->ip;
5374         struct meter *m;
5375         uint64_t time, n_pkts, n_bytes;
5376         uint32_t length;
5377         enum rte_color color_in, color_out;
5378
5379         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
5380
5381         /* Structs. */
5382         m = instr_meter_idx_nbo(p, t, ip);
5383         rte_prefetch0(m->n_pkts);
5384         time = rte_get_tsc_cycles();
5385         length = instr_meter_length_hbo(t, ip);
5386         color_in = (enum rte_color)ip->meter.color_in_val;
5387
5388         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5389                 &m->profile->profile,
5390                 time,
5391                 length,
5392                 color_in);
5393
5394         color_out &= m->color_mask;
5395
5396         n_pkts = m->n_pkts[color_out];
5397         n_bytes = m->n_bytes[color_out];
5398
5399         instr_meter_color_out_hbo_set(t, ip, color_out);
5400
5401         m->n_pkts[color_out] = n_pkts + 1;
5402         m->n_bytes[color_out] = n_bytes + length;
5403
5404         /* Thread. */
5405         thread_ip_inc(p);
5406 }
5407
5408 static inline void
5409 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5410 {
5411         struct thread *t = &p->threads[p->thread_id];
5412         struct instruction *ip = t->ip;
5413         struct meter *m;
5414         uint64_t time, n_pkts, n_bytes;
5415         uint32_t length;
5416         enum rte_color color_in, color_out;
5417
5418         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
5419
5420         /* Structs. */
5421         m = instr_meter_idx_hbo(p, t, ip);
5422         rte_prefetch0(m->n_pkts);
5423         time = rte_get_tsc_cycles();
5424         length = instr_meter_length_nbo(t, ip);
5425         color_in = instr_meter_color_in_hbo(t, ip);
5426
5427         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5428                 &m->profile->profile,
5429                 time,
5430                 length,
5431                 color_in);
5432
5433         color_out &= m->color_mask;
5434
5435         n_pkts = m->n_pkts[color_out];
5436         n_bytes = m->n_bytes[color_out];
5437
5438         instr_meter_color_out_hbo_set(t, ip, color_out);
5439
5440         m->n_pkts[color_out] = n_pkts + 1;
5441         m->n_bytes[color_out] = n_bytes + length;
5442
5443         /* Thread. */
5444         thread_ip_inc(p);
5445 }
5446
5447 static inline void
5448 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5449 {
5450         struct thread *t = &p->threads[p->thread_id];
5451         struct instruction *ip = t->ip;
5452         struct meter *m;
5453         uint64_t time, n_pkts, n_bytes;
5454         uint32_t length;
5455         enum rte_color color_in, color_out;
5456
5457         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
5458
5459         /* Structs. */
5460         m = instr_meter_idx_hbo(p, t, ip);
5461         rte_prefetch0(m->n_pkts);
5462         time = rte_get_tsc_cycles();
5463         length = instr_meter_length_nbo(t, ip);
5464         color_in = (enum rte_color)ip->meter.color_in_val;
5465
5466         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5467                 &m->profile->profile,
5468                 time,
5469                 length,
5470                 color_in);
5471
5472         color_out &= m->color_mask;
5473
5474         n_pkts = m->n_pkts[color_out];
5475         n_bytes = m->n_bytes[color_out];
5476
5477         instr_meter_color_out_hbo_set(t, ip, color_out);
5478
5479         m->n_pkts[color_out] = n_pkts + 1;
5480         m->n_bytes[color_out] = n_bytes + length;
5481
5482         /* Thread. */
5483         thread_ip_inc(p);
5484 }
5485
5486 static inline void
5487 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5488 {
5489         struct thread *t = &p->threads[p->thread_id];
5490         struct instruction *ip = t->ip;
5491         struct meter *m;
5492         uint64_t time, n_pkts, n_bytes;
5493         uint32_t length;
5494         enum rte_color color_in, color_out;
5495
5496         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
5497
5498         /* Structs. */
5499         m = instr_meter_idx_hbo(p, t, ip);
5500         rte_prefetch0(m->n_pkts);
5501         time = rte_get_tsc_cycles();
5502         length = instr_meter_length_hbo(t, ip);
5503         color_in = instr_meter_color_in_hbo(t, ip);
5504
5505         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5506                 &m->profile->profile,
5507                 time,
5508                 length,
5509                 color_in);
5510
5511         color_out &= m->color_mask;
5512
5513         n_pkts = m->n_pkts[color_out];
5514         n_bytes = m->n_bytes[color_out];
5515
5516         instr_meter_color_out_hbo_set(t, ip, color_out);
5517
5518         m->n_pkts[color_out] = n_pkts + 1;
5519         m->n_bytes[color_out] = n_bytes + length;
5520
5521         /* Thread. */
5522         thread_ip_inc(p);
5523 }
5524
5525 static inline void
5526 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5527 {
5528         struct thread *t = &p->threads[p->thread_id];
5529         struct instruction *ip = t->ip;
5530         struct meter *m;
5531         uint64_t time, n_pkts, n_bytes;
5532         uint32_t length;
5533         enum rte_color color_in, color_out;
5534
5535         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
5536
5537         /* Structs. */
5538         m = instr_meter_idx_hbo(p, t, ip);
5539         rte_prefetch0(m->n_pkts);
5540         time = rte_get_tsc_cycles();
5541         length = instr_meter_length_hbo(t, ip);
5542         color_in = (enum rte_color)ip->meter.color_in_val;
5543
5544         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5545                 &m->profile->profile,
5546                 time,
5547                 length,
5548                 color_in);
5549
5550         color_out &= m->color_mask;
5551
5552         n_pkts = m->n_pkts[color_out];
5553         n_bytes = m->n_bytes[color_out];
5554
5555         instr_meter_color_out_hbo_set(t, ip, color_out);
5556
5557         m->n_pkts[color_out] = n_pkts + 1;
5558         m->n_bytes[color_out] = n_bytes + length;
5559
5560         /* Thread. */
5561         thread_ip_inc(p);
5562 }
5563
5564 static inline void
5565 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5566 {
5567         struct thread *t = &p->threads[p->thread_id];
5568         struct instruction *ip = t->ip;
5569         struct meter *m;
5570         uint64_t time, n_pkts, n_bytes;
5571         uint32_t length;
5572         enum rte_color color_in, color_out;
5573
5574         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
5575
5576         /* Structs. */
5577         m = instr_meter_idx_imm(p, ip);
5578         rte_prefetch0(m->n_pkts);
5579         time = rte_get_tsc_cycles();
5580         length = instr_meter_length_nbo(t, ip);
5581         color_in = instr_meter_color_in_hbo(t, ip);
5582
5583         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5584                 &m->profile->profile,
5585                 time,
5586                 length,
5587                 color_in);
5588
5589         color_out &= m->color_mask;
5590
5591         n_pkts = m->n_pkts[color_out];
5592         n_bytes = m->n_bytes[color_out];
5593
5594         instr_meter_color_out_hbo_set(t, ip, color_out);
5595
5596         m->n_pkts[color_out] = n_pkts + 1;
5597         m->n_bytes[color_out] = n_bytes + length;
5598
5599         /* Thread. */
5600         thread_ip_inc(p);
5601 }
5602
5603 static inline void
5604 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5605 {
5606         struct thread *t = &p->threads[p->thread_id];
5607         struct instruction *ip = t->ip;
5608         struct meter *m;
5609         uint64_t time, n_pkts, n_bytes;
5610         uint32_t length;
5611         enum rte_color color_in, color_out;
5612
5613         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
5614
5615         /* Structs. */
5616         m = instr_meter_idx_imm(p, ip);
5617         rte_prefetch0(m->n_pkts);
5618         time = rte_get_tsc_cycles();
5619         length = instr_meter_length_nbo(t, ip);
5620         color_in = (enum rte_color)ip->meter.color_in_val;
5621
5622         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5623                 &m->profile->profile,
5624                 time,
5625                 length,
5626                 color_in);
5627
5628         color_out &= m->color_mask;
5629
5630         n_pkts = m->n_pkts[color_out];
5631         n_bytes = m->n_bytes[color_out];
5632
5633         instr_meter_color_out_hbo_set(t, ip, color_out);
5634
5635         m->n_pkts[color_out] = n_pkts + 1;
5636         m->n_bytes[color_out] = n_bytes + length;
5637
5638         /* Thread. */
5639         thread_ip_inc(p);
5640 }
5641
5642 static inline void
5643 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5644 {
5645         struct thread *t = &p->threads[p->thread_id];
5646         struct instruction *ip = t->ip;
5647         struct meter *m;
5648         uint64_t time, n_pkts, n_bytes;
5649         uint32_t length;
5650         enum rte_color color_in, color_out;
5651
5652         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
5653
5654         /* Structs. */
5655         m = instr_meter_idx_imm(p, ip);
5656         rte_prefetch0(m->n_pkts);
5657         time = rte_get_tsc_cycles();
5658         length = instr_meter_length_hbo(t, ip);
5659         color_in = instr_meter_color_in_hbo(t, ip);
5660
5661         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5662                 &m->profile->profile,
5663                 time,
5664                 length,
5665                 color_in);
5666
5667         color_out &= m->color_mask;
5668
5669         n_pkts = m->n_pkts[color_out];
5670         n_bytes = m->n_bytes[color_out];
5671
5672         instr_meter_color_out_hbo_set(t, ip, color_out);
5673
5674         m->n_pkts[color_out] = n_pkts + 1;
5675         m->n_bytes[color_out] = n_bytes + length;
5676
5677         /* Thread. */
5678         thread_ip_inc(p);
5679 }
5680 static inline void
5681 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5682 {
5683         struct thread *t = &p->threads[p->thread_id];
5684         struct instruction *ip = t->ip;
5685         struct meter *m;
5686         uint64_t time, n_pkts, n_bytes;
5687         uint32_t length;
5688         enum rte_color color_in, color_out;
5689
5690         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
5691
5692         /* Structs. */
5693         m = instr_meter_idx_imm(p, ip);
5694         rte_prefetch0(m->n_pkts);
5695         time = rte_get_tsc_cycles();
5696         length = instr_meter_length_hbo(t, ip);
5697         color_in = (enum rte_color)ip->meter.color_in_val;
5698
5699         color_out = rte_meter_trtcm_color_aware_check(&m->m,
5700                 &m->profile->profile,
5701                 time,
5702                 length,
5703                 color_in);
5704
5705         color_out &= m->color_mask;
5706
5707         n_pkts = m->n_pkts[color_out];
5708         n_bytes = m->n_bytes[color_out];
5709
5710         instr_meter_color_out_hbo_set(t, ip, color_out);
5711
5712         m->n_pkts[color_out] = n_pkts + 1;
5713         m->n_bytes[color_out] = n_bytes + length;
5714
5715         /* Thread. */
5716         thread_ip_inc(p);
5717 }
5718
5719 /*
5720  * jmp.
5721  */
5722 static int
5723 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5724                     struct action *action __rte_unused,
5725                     char **tokens,
5726                     int n_tokens,
5727                     struct instruction *instr,
5728                     struct instruction_data *data)
5729 {
5730         CHECK(n_tokens == 2, EINVAL);
5731
5732         strcpy(data->jmp_label, tokens[1]);
5733
5734         instr->type = INSTR_JMP;
5735         instr->jmp.ip = NULL; /* Resolved later. */
5736         return 0;
5737 }
5738
5739 static int
5740 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5741                           struct action *action __rte_unused,
5742                           char **tokens,
5743                           int n_tokens,
5744                           struct instruction *instr,
5745                           struct instruction_data *data)
5746 {
5747         struct header *h;
5748
5749         CHECK(n_tokens == 3, EINVAL);
5750
5751         strcpy(data->jmp_label, tokens[1]);
5752
5753         h = header_parse(p, tokens[2]);
5754         CHECK(h, EINVAL);
5755
5756         instr->type = INSTR_JMP_VALID;
5757         instr->jmp.ip = NULL; /* Resolved later. */
5758         instr->jmp.header_id = h->id;
5759         return 0;
5760 }
5761
5762 static int
5763 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5764                             struct action *action __rte_unused,
5765                             char **tokens,
5766                             int n_tokens,
5767                             struct instruction *instr,
5768                             struct instruction_data *data)
5769 {
5770         struct header *h;
5771
5772         CHECK(n_tokens == 3, EINVAL);
5773
5774         strcpy(data->jmp_label, tokens[1]);
5775
5776         h = header_parse(p, tokens[2]);
5777         CHECK(h, EINVAL);
5778
5779         instr->type = INSTR_JMP_INVALID;
5780         instr->jmp.ip = NULL; /* Resolved later. */
5781         instr->jmp.header_id = h->id;
5782         return 0;
5783 }
5784
5785 static int
5786 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5787                         struct action *action,
5788                         char **tokens,
5789                         int n_tokens,
5790                         struct instruction *instr,
5791                         struct instruction_data *data)
5792 {
5793         CHECK(!action, EINVAL);
5794         CHECK(n_tokens == 2, EINVAL);
5795
5796         strcpy(data->jmp_label, tokens[1]);
5797
5798         instr->type = INSTR_JMP_HIT;
5799         instr->jmp.ip = NULL; /* Resolved later. */
5800         return 0;
5801 }
5802
5803 static int
5804 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5805                          struct action *action,
5806                          char **tokens,
5807                          int n_tokens,
5808                          struct instruction *instr,
5809                          struct instruction_data *data)
5810 {
5811         CHECK(!action, EINVAL);
5812         CHECK(n_tokens == 2, EINVAL);
5813
5814         strcpy(data->jmp_label, tokens[1]);
5815
5816         instr->type = INSTR_JMP_MISS;
5817         instr->jmp.ip = NULL; /* Resolved later. */
5818         return 0;
5819 }
5820
5821 static int
5822 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5823                                struct action *action,
5824                                char **tokens,
5825                                int n_tokens,
5826                                struct instruction *instr,
5827                                struct instruction_data *data)
5828 {
5829         struct action *a;
5830
5831         CHECK(!action, EINVAL);
5832         CHECK(n_tokens == 3, EINVAL);
5833
5834         strcpy(data->jmp_label, tokens[1]);
5835
5836         a = action_find(p, tokens[2]);
5837         CHECK(a, EINVAL);
5838
5839         instr->type = INSTR_JMP_ACTION_HIT;
5840         instr->jmp.ip = NULL; /* Resolved later. */
5841         instr->jmp.action_id = a->id;
5842         return 0;
5843 }
5844
5845 static int
5846 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5847                                 struct action *action,
5848                                 char **tokens,
5849                                 int n_tokens,
5850                                 struct instruction *instr,
5851                                 struct instruction_data *data)
5852 {
5853         struct action *a;
5854
5855         CHECK(!action, EINVAL);
5856         CHECK(n_tokens == 3, EINVAL);
5857
5858         strcpy(data->jmp_label, tokens[1]);
5859
5860         a = action_find(p, tokens[2]);
5861         CHECK(a, EINVAL);
5862
5863         instr->type = INSTR_JMP_ACTION_MISS;
5864         instr->jmp.ip = NULL; /* Resolved later. */
5865         instr->jmp.action_id = a->id;
5866         return 0;
5867 }
5868
5869 static int
5870 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5871                        struct action *action,
5872                        char **tokens,
5873                        int n_tokens,
5874                        struct instruction *instr,
5875                        struct instruction_data *data)
5876 {
5877         char *a = tokens[2], *b = tokens[3];
5878         struct field *fa, *fb;
5879         uint64_t b_val;
5880         uint32_t a_struct_id, b_struct_id;
5881
5882         CHECK(n_tokens == 4, EINVAL);
5883
5884         strcpy(data->jmp_label, tokens[1]);
5885
5886         fa = struct_field_parse(p, action, a, &a_struct_id);
5887         CHECK(fa, EINVAL);
5888         CHECK(!fa->var_size, EINVAL);
5889
5890         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5891         fb = struct_field_parse(p, action, b, &b_struct_id);
5892         if (fb) {
5893                 CHECK(!fb->var_size, EINVAL);
5894
5895                 instr->type = INSTR_JMP_EQ;
5896                 if (a[0] != 'h' && b[0] == 'h')
5897                         instr->type = INSTR_JMP_EQ_MH;
5898                 if (a[0] == 'h' && b[0] != 'h')
5899                         instr->type = INSTR_JMP_EQ_HM;
5900                 if (a[0] == 'h' && b[0] == 'h')
5901                         instr->type = INSTR_JMP_EQ_HH;
5902                 instr->jmp.ip = NULL; /* Resolved later. */
5903
5904                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5905                 instr->jmp.a.n_bits = fa->n_bits;
5906                 instr->jmp.a.offset = fa->offset / 8;
5907                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5908                 instr->jmp.b.n_bits = fb->n_bits;
5909                 instr->jmp.b.offset = fb->offset / 8;
5910                 return 0;
5911         }
5912
5913         /* JMP_EQ_I. */
5914         b_val = strtoull(b, &b, 0);
5915         CHECK(!b[0], EINVAL);
5916
5917         if (a[0] == 'h')
5918                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5919
5920         instr->type = INSTR_JMP_EQ_I;
5921         instr->jmp.ip = NULL; /* Resolved later. */
5922         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5923         instr->jmp.a.n_bits = fa->n_bits;
5924         instr->jmp.a.offset = fa->offset / 8;
5925         instr->jmp.b_val = b_val;
5926         return 0;
5927 }
5928
5929 static int
5930 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5931                         struct action *action,
5932                         char **tokens,
5933                         int n_tokens,
5934                         struct instruction *instr,
5935                         struct instruction_data *data)
5936 {
5937         char *a = tokens[2], *b = tokens[3];
5938         struct field *fa, *fb;
5939         uint64_t b_val;
5940         uint32_t a_struct_id, b_struct_id;
5941
5942         CHECK(n_tokens == 4, EINVAL);
5943
5944         strcpy(data->jmp_label, tokens[1]);
5945
5946         fa = struct_field_parse(p, action, a, &a_struct_id);
5947         CHECK(fa, EINVAL);
5948         CHECK(!fa->var_size, EINVAL);
5949
5950         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5951         fb = struct_field_parse(p, action, b, &b_struct_id);
5952         if (fb) {
5953                 CHECK(!fb->var_size, EINVAL);
5954
5955                 instr->type = INSTR_JMP_NEQ;
5956                 if (a[0] != 'h' && b[0] == 'h')
5957                         instr->type = INSTR_JMP_NEQ_MH;
5958                 if (a[0] == 'h' && b[0] != 'h')
5959                         instr->type = INSTR_JMP_NEQ_HM;
5960                 if (a[0] == 'h' && b[0] == 'h')
5961                         instr->type = INSTR_JMP_NEQ_HH;
5962                 instr->jmp.ip = NULL; /* Resolved later. */
5963
5964                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5965                 instr->jmp.a.n_bits = fa->n_bits;
5966                 instr->jmp.a.offset = fa->offset / 8;
5967                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5968                 instr->jmp.b.n_bits = fb->n_bits;
5969                 instr->jmp.b.offset = fb->offset / 8;
5970                 return 0;
5971         }
5972
5973         /* JMP_NEQ_I. */
5974         b_val = strtoull(b, &b, 0);
5975         CHECK(!b[0], EINVAL);
5976
5977         if (a[0] == 'h')
5978                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5979
5980         instr->type = INSTR_JMP_NEQ_I;
5981         instr->jmp.ip = NULL; /* Resolved later. */
5982         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5983         instr->jmp.a.n_bits = fa->n_bits;
5984         instr->jmp.a.offset = fa->offset / 8;
5985         instr->jmp.b_val = b_val;
5986         return 0;
5987 }
5988
5989 static int
5990 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5991                        struct action *action,
5992                        char **tokens,
5993                        int n_tokens,
5994                        struct instruction *instr,
5995                        struct instruction_data *data)
5996 {
5997         char *a = tokens[2], *b = tokens[3];
5998         struct field *fa, *fb;
5999         uint64_t b_val;
6000         uint32_t a_struct_id, b_struct_id;
6001
6002         CHECK(n_tokens == 4, EINVAL);
6003
6004         strcpy(data->jmp_label, tokens[1]);
6005
6006         fa = struct_field_parse(p, action, a, &a_struct_id);
6007         CHECK(fa, EINVAL);
6008         CHECK(!fa->var_size, EINVAL);
6009
6010         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6011         fb = struct_field_parse(p, action, b, &b_struct_id);
6012         if (fb) {
6013                 CHECK(!fb->var_size, EINVAL);
6014
6015                 instr->type = INSTR_JMP_LT;
6016                 if (a[0] == 'h' && b[0] != 'h')
6017                         instr->type = INSTR_JMP_LT_HM;
6018                 if (a[0] != 'h' && b[0] == 'h')
6019                         instr->type = INSTR_JMP_LT_MH;
6020                 if (a[0] == 'h' && b[0] == 'h')
6021                         instr->type = INSTR_JMP_LT_HH;
6022                 instr->jmp.ip = NULL; /* Resolved later. */
6023
6024                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6025                 instr->jmp.a.n_bits = fa->n_bits;
6026                 instr->jmp.a.offset = fa->offset / 8;
6027                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6028                 instr->jmp.b.n_bits = fb->n_bits;
6029                 instr->jmp.b.offset = fb->offset / 8;
6030                 return 0;
6031         }
6032
6033         /* JMP_LT_MI, JMP_LT_HI. */
6034         b_val = strtoull(b, &b, 0);
6035         CHECK(!b[0], EINVAL);
6036
6037         instr->type = INSTR_JMP_LT_MI;
6038         if (a[0] == 'h')
6039                 instr->type = INSTR_JMP_LT_HI;
6040         instr->jmp.ip = NULL; /* Resolved later. */
6041
6042         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6043         instr->jmp.a.n_bits = fa->n_bits;
6044         instr->jmp.a.offset = fa->offset / 8;
6045         instr->jmp.b_val = b_val;
6046         return 0;
6047 }
6048
6049 static int
6050 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6051                        struct action *action,
6052                        char **tokens,
6053                        int n_tokens,
6054                        struct instruction *instr,
6055                        struct instruction_data *data)
6056 {
6057         char *a = tokens[2], *b = tokens[3];
6058         struct field *fa, *fb;
6059         uint64_t b_val;
6060         uint32_t a_struct_id, b_struct_id;
6061
6062         CHECK(n_tokens == 4, EINVAL);
6063
6064         strcpy(data->jmp_label, tokens[1]);
6065
6066         fa = struct_field_parse(p, action, a, &a_struct_id);
6067         CHECK(fa, EINVAL);
6068         CHECK(!fa->var_size, EINVAL);
6069
6070         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6071         fb = struct_field_parse(p, action, b, &b_struct_id);
6072         if (fb) {
6073                 CHECK(!fb->var_size, EINVAL);
6074
6075                 instr->type = INSTR_JMP_GT;
6076                 if (a[0] == 'h' && b[0] != 'h')
6077                         instr->type = INSTR_JMP_GT_HM;
6078                 if (a[0] != 'h' && b[0] == 'h')
6079                         instr->type = INSTR_JMP_GT_MH;
6080                 if (a[0] == 'h' && b[0] == 'h')
6081                         instr->type = INSTR_JMP_GT_HH;
6082                 instr->jmp.ip = NULL; /* Resolved later. */
6083
6084                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6085                 instr->jmp.a.n_bits = fa->n_bits;
6086                 instr->jmp.a.offset = fa->offset / 8;
6087                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6088                 instr->jmp.b.n_bits = fb->n_bits;
6089                 instr->jmp.b.offset = fb->offset / 8;
6090                 return 0;
6091         }
6092
6093         /* JMP_GT_MI, JMP_GT_HI. */
6094         b_val = strtoull(b, &b, 0);
6095         CHECK(!b[0], EINVAL);
6096
6097         instr->type = INSTR_JMP_GT_MI;
6098         if (a[0] == 'h')
6099                 instr->type = INSTR_JMP_GT_HI;
6100         instr->jmp.ip = NULL; /* Resolved later. */
6101
6102         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6103         instr->jmp.a.n_bits = fa->n_bits;
6104         instr->jmp.a.offset = fa->offset / 8;
6105         instr->jmp.b_val = b_val;
6106         return 0;
6107 }
6108
6109 static inline void
6110 instr_jmp_exec(struct rte_swx_pipeline *p)
6111 {
6112         struct thread *t = &p->threads[p->thread_id];
6113         struct instruction *ip = t->ip;
6114
6115         TRACE("[Thread %2u] jmp\n", p->thread_id);
6116
6117         thread_ip_set(t, ip->jmp.ip);
6118 }
6119
6120 static inline void
6121 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6122 {
6123         struct thread *t = &p->threads[p->thread_id];
6124         struct instruction *ip = t->ip;
6125         uint32_t header_id = ip->jmp.header_id;
6126
6127         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6128
6129         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6130 }
6131
6132 static inline void
6133 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6134 {
6135         struct thread *t = &p->threads[p->thread_id];
6136         struct instruction *ip = t->ip;
6137         uint32_t header_id = ip->jmp.header_id;
6138
6139         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6140
6141         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6142 }
6143
6144 static inline void
6145 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6146 {
6147         struct thread *t = &p->threads[p->thread_id];
6148         struct instruction *ip = t->ip;
6149         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6150
6151         TRACE("[Thread %2u] jmph\n", p->thread_id);
6152
6153         t->ip = ip_next[t->hit];
6154 }
6155
6156 static inline void
6157 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6158 {
6159         struct thread *t = &p->threads[p->thread_id];
6160         struct instruction *ip = t->ip;
6161         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6162
6163         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
6164
6165         t->ip = ip_next[t->hit];
6166 }
6167
6168 static inline void
6169 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
6170 {
6171         struct thread *t = &p->threads[p->thread_id];
6172         struct instruction *ip = t->ip;
6173
6174         TRACE("[Thread %2u] jmpa\n", p->thread_id);
6175
6176         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
6177 }
6178
6179 static inline void
6180 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
6181 {
6182         struct thread *t = &p->threads[p->thread_id];
6183         struct instruction *ip = t->ip;
6184
6185         TRACE("[Thread %2u] jmpna\n", p->thread_id);
6186
6187         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
6188 }
6189
6190 static inline void
6191 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
6192 {
6193         struct thread *t = &p->threads[p->thread_id];
6194         struct instruction *ip = t->ip;
6195
6196         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
6197
6198         JMP_CMP(t, ip, ==);
6199 }
6200
6201 static inline void
6202 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
6203 {
6204         struct thread *t = &p->threads[p->thread_id];
6205         struct instruction *ip = t->ip;
6206
6207         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
6208
6209         JMP_CMP_MH(t, ip, ==);
6210 }
6211
6212 static inline void
6213 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
6214 {
6215         struct thread *t = &p->threads[p->thread_id];
6216         struct instruction *ip = t->ip;
6217
6218         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
6219
6220         JMP_CMP_HM(t, ip, ==);
6221 }
6222
6223 static inline void
6224 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
6225 {
6226         struct thread *t = &p->threads[p->thread_id];
6227         struct instruction *ip = t->ip;
6228
6229         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
6230
6231         JMP_CMP_HH_FAST(t, ip, ==);
6232 }
6233
6234 static inline void
6235 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
6236 {
6237         struct thread *t = &p->threads[p->thread_id];
6238         struct instruction *ip = t->ip;
6239
6240         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
6241
6242         JMP_CMP_I(t, ip, ==);
6243 }
6244
6245 static inline void
6246 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
6247 {
6248         struct thread *t = &p->threads[p->thread_id];
6249         struct instruction *ip = t->ip;
6250
6251         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
6252
6253         JMP_CMP(t, ip, !=);
6254 }
6255
6256 static inline void
6257 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
6258 {
6259         struct thread *t = &p->threads[p->thread_id];
6260         struct instruction *ip = t->ip;
6261
6262         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
6263
6264         JMP_CMP_MH(t, ip, !=);
6265 }
6266
6267 static inline void
6268 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
6269 {
6270         struct thread *t = &p->threads[p->thread_id];
6271         struct instruction *ip = t->ip;
6272
6273         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
6274
6275         JMP_CMP_HM(t, ip, !=);
6276 }
6277
6278 static inline void
6279 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
6280 {
6281         struct thread *t = &p->threads[p->thread_id];
6282         struct instruction *ip = t->ip;
6283
6284         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
6285
6286         JMP_CMP_HH_FAST(t, ip, !=);
6287 }
6288
6289 static inline void
6290 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
6291 {
6292         struct thread *t = &p->threads[p->thread_id];
6293         struct instruction *ip = t->ip;
6294
6295         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
6296
6297         JMP_CMP_I(t, ip, !=);
6298 }
6299
6300 static inline void
6301 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
6302 {
6303         struct thread *t = &p->threads[p->thread_id];
6304         struct instruction *ip = t->ip;
6305
6306         TRACE("[Thread %2u] jmplt\n", p->thread_id);
6307
6308         JMP_CMP(t, ip, <);
6309 }
6310
6311 static inline void
6312 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
6313 {
6314         struct thread *t = &p->threads[p->thread_id];
6315         struct instruction *ip = t->ip;
6316
6317         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
6318
6319         JMP_CMP_MH(t, ip, <);
6320 }
6321
6322 static inline void
6323 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
6324 {
6325         struct thread *t = &p->threads[p->thread_id];
6326         struct instruction *ip = t->ip;
6327
6328         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
6329
6330         JMP_CMP_HM(t, ip, <);
6331 }
6332
6333 static inline void
6334 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
6335 {
6336         struct thread *t = &p->threads[p->thread_id];
6337         struct instruction *ip = t->ip;
6338
6339         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
6340
6341         JMP_CMP_HH(t, ip, <);
6342 }
6343
6344 static inline void
6345 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
6346 {
6347         struct thread *t = &p->threads[p->thread_id];
6348         struct instruction *ip = t->ip;
6349
6350         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
6351
6352         JMP_CMP_MI(t, ip, <);
6353 }
6354
6355 static inline void
6356 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
6357 {
6358         struct thread *t = &p->threads[p->thread_id];
6359         struct instruction *ip = t->ip;
6360
6361         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
6362
6363         JMP_CMP_HI(t, ip, <);
6364 }
6365
6366 static inline void
6367 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
6368 {
6369         struct thread *t = &p->threads[p->thread_id];
6370         struct instruction *ip = t->ip;
6371
6372         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
6373
6374         JMP_CMP(t, ip, >);
6375 }
6376
6377 static inline void
6378 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
6379 {
6380         struct thread *t = &p->threads[p->thread_id];
6381         struct instruction *ip = t->ip;
6382
6383         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
6384
6385         JMP_CMP_MH(t, ip, >);
6386 }
6387
6388 static inline void
6389 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
6390 {
6391         struct thread *t = &p->threads[p->thread_id];
6392         struct instruction *ip = t->ip;
6393
6394         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
6395
6396         JMP_CMP_HM(t, ip, >);
6397 }
6398
6399 static inline void
6400 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
6401 {
6402         struct thread *t = &p->threads[p->thread_id];
6403         struct instruction *ip = t->ip;
6404
6405         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
6406
6407         JMP_CMP_HH(t, ip, >);
6408 }
6409
6410 static inline void
6411 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
6412 {
6413         struct thread *t = &p->threads[p->thread_id];
6414         struct instruction *ip = t->ip;
6415
6416         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
6417
6418         JMP_CMP_MI(t, ip, >);
6419 }
6420
6421 static inline void
6422 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
6423 {
6424         struct thread *t = &p->threads[p->thread_id];
6425         struct instruction *ip = t->ip;
6426
6427         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
6428
6429         JMP_CMP_HI(t, ip, >);
6430 }
6431
6432 /*
6433  * return.
6434  */
6435 static int
6436 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
6437                        struct action *action,
6438                        char **tokens __rte_unused,
6439                        int n_tokens,
6440                        struct instruction *instr,
6441                        struct instruction_data *data __rte_unused)
6442 {
6443         CHECK(action, EINVAL);
6444         CHECK(n_tokens == 1, EINVAL);
6445
6446         instr->type = INSTR_RETURN;
6447         return 0;
6448 }
6449
6450 static inline void
6451 instr_return_exec(struct rte_swx_pipeline *p)
6452 {
6453         struct thread *t = &p->threads[p->thread_id];
6454
6455         TRACE("[Thread %2u] return\n", p->thread_id);
6456
6457         t->ip = t->ret;
6458 }
6459
6460 static int
6461 instr_translate(struct rte_swx_pipeline *p,
6462                 struct action *action,
6463                 char *string,
6464                 struct instruction *instr,
6465                 struct instruction_data *data)
6466 {
6467         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6468         int n_tokens = 0, tpos = 0;
6469
6470         /* Parse the instruction string into tokens. */
6471         for ( ; ; ) {
6472                 char *token;
6473
6474                 token = strtok_r(string, " \t\v", &string);
6475                 if (!token)
6476                         break;
6477
6478                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6479                 CHECK_NAME(token, EINVAL);
6480
6481                 tokens[n_tokens] = token;
6482                 n_tokens++;
6483         }
6484
6485         CHECK(n_tokens, EINVAL);
6486
6487         /* Handle the optional instruction label. */
6488         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6489                 strcpy(data->label, tokens[0]);
6490
6491                 tpos += 2;
6492                 CHECK(n_tokens - tpos, EINVAL);
6493         }
6494
6495         /* Identify the instruction type. */
6496         if (!strcmp(tokens[tpos], "rx"))
6497                 return instr_rx_translate(p,
6498                                           action,
6499                                           &tokens[tpos],
6500                                           n_tokens - tpos,
6501                                           instr,
6502                                           data);
6503
6504         if (!strcmp(tokens[tpos], "tx"))
6505                 return instr_tx_translate(p,
6506                                           action,
6507                                           &tokens[tpos],
6508                                           n_tokens - tpos,
6509                                           instr,
6510                                           data);
6511
6512         if (!strcmp(tokens[tpos], "drop"))
6513                 return instr_drop_translate(p,
6514                                             action,
6515                                             &tokens[tpos],
6516                                             n_tokens - tpos,
6517                                             instr,
6518                                             data);
6519
6520         if (!strcmp(tokens[tpos], "extract"))
6521                 return instr_hdr_extract_translate(p,
6522                                                    action,
6523                                                    &tokens[tpos],
6524                                                    n_tokens - tpos,
6525                                                    instr,
6526                                                    data);
6527
6528         if (!strcmp(tokens[tpos], "lookahead"))
6529                 return instr_hdr_lookahead_translate(p,
6530                                                      action,
6531                                                      &tokens[tpos],
6532                                                      n_tokens - tpos,
6533                                                      instr,
6534                                                      data);
6535
6536         if (!strcmp(tokens[tpos], "emit"))
6537                 return instr_hdr_emit_translate(p,
6538                                                 action,
6539                                                 &tokens[tpos],
6540                                                 n_tokens - tpos,
6541                                                 instr,
6542                                                 data);
6543
6544         if (!strcmp(tokens[tpos], "validate"))
6545                 return instr_hdr_validate_translate(p,
6546                                                     action,
6547                                                     &tokens[tpos],
6548                                                     n_tokens - tpos,
6549                                                     instr,
6550                                                     data);
6551
6552         if (!strcmp(tokens[tpos], "invalidate"))
6553                 return instr_hdr_invalidate_translate(p,
6554                                                       action,
6555                                                       &tokens[tpos],
6556                                                       n_tokens - tpos,
6557                                                       instr,
6558                                                       data);
6559
6560         if (!strcmp(tokens[tpos], "mov"))
6561                 return instr_mov_translate(p,
6562                                            action,
6563                                            &tokens[tpos],
6564                                            n_tokens - tpos,
6565                                            instr,
6566                                            data);
6567
6568         if (!strcmp(tokens[tpos], "add"))
6569                 return instr_alu_add_translate(p,
6570                                                action,
6571                                                &tokens[tpos],
6572                                                n_tokens - tpos,
6573                                                instr,
6574                                                data);
6575
6576         if (!strcmp(tokens[tpos], "sub"))
6577                 return instr_alu_sub_translate(p,
6578                                                action,
6579                                                &tokens[tpos],
6580                                                n_tokens - tpos,
6581                                                instr,
6582                                                data);
6583
6584         if (!strcmp(tokens[tpos], "ckadd"))
6585                 return instr_alu_ckadd_translate(p,
6586                                                  action,
6587                                                  &tokens[tpos],
6588                                                  n_tokens - tpos,
6589                                                  instr,
6590                                                  data);
6591
6592         if (!strcmp(tokens[tpos], "cksub"))
6593                 return instr_alu_cksub_translate(p,
6594                                                  action,
6595                                                  &tokens[tpos],
6596                                                  n_tokens - tpos,
6597                                                  instr,
6598                                                  data);
6599
6600         if (!strcmp(tokens[tpos], "and"))
6601                 return instr_alu_and_translate(p,
6602                                                action,
6603                                                &tokens[tpos],
6604                                                n_tokens - tpos,
6605                                                instr,
6606                                                data);
6607
6608         if (!strcmp(tokens[tpos], "or"))
6609                 return instr_alu_or_translate(p,
6610                                               action,
6611                                               &tokens[tpos],
6612                                               n_tokens - tpos,
6613                                               instr,
6614                                               data);
6615
6616         if (!strcmp(tokens[tpos], "xor"))
6617                 return instr_alu_xor_translate(p,
6618                                                action,
6619                                                &tokens[tpos],
6620                                                n_tokens - tpos,
6621                                                instr,
6622                                                data);
6623
6624         if (!strcmp(tokens[tpos], "shl"))
6625                 return instr_alu_shl_translate(p,
6626                                                action,
6627                                                &tokens[tpos],
6628                                                n_tokens - tpos,
6629                                                instr,
6630                                                data);
6631
6632         if (!strcmp(tokens[tpos], "shr"))
6633                 return instr_alu_shr_translate(p,
6634                                                action,
6635                                                &tokens[tpos],
6636                                                n_tokens - tpos,
6637                                                instr,
6638                                                data);
6639
6640         if (!strcmp(tokens[tpos], "regprefetch"))
6641                 return instr_regprefetch_translate(p,
6642                                                    action,
6643                                                    &tokens[tpos],
6644                                                    n_tokens - tpos,
6645                                                    instr,
6646                                                    data);
6647
6648         if (!strcmp(tokens[tpos], "regrd"))
6649                 return instr_regrd_translate(p,
6650                                              action,
6651                                              &tokens[tpos],
6652                                              n_tokens - tpos,
6653                                              instr,
6654                                              data);
6655
6656         if (!strcmp(tokens[tpos], "regwr"))
6657                 return instr_regwr_translate(p,
6658                                              action,
6659                                              &tokens[tpos],
6660                                              n_tokens - tpos,
6661                                              instr,
6662                                              data);
6663
6664         if (!strcmp(tokens[tpos], "regadd"))
6665                 return instr_regadd_translate(p,
6666                                               action,
6667                                               &tokens[tpos],
6668                                               n_tokens - tpos,
6669                                               instr,
6670                                               data);
6671
6672         if (!strcmp(tokens[tpos], "metprefetch"))
6673                 return instr_metprefetch_translate(p,
6674                                                    action,
6675                                                    &tokens[tpos],
6676                                                    n_tokens - tpos,
6677                                                    instr,
6678                                                    data);
6679
6680         if (!strcmp(tokens[tpos], "meter"))
6681                 return instr_meter_translate(p,
6682                                              action,
6683                                              &tokens[tpos],
6684                                              n_tokens - tpos,
6685                                              instr,
6686                                              data);
6687
6688         if (!strcmp(tokens[tpos], "table"))
6689                 return instr_table_translate(p,
6690                                              action,
6691                                              &tokens[tpos],
6692                                              n_tokens - tpos,
6693                                              instr,
6694                                              data);
6695
6696         if (!strcmp(tokens[tpos], "learn"))
6697                 return instr_learn_translate(p,
6698                                              action,
6699                                              &tokens[tpos],
6700                                              n_tokens - tpos,
6701                                              instr,
6702                                              data);
6703
6704         if (!strcmp(tokens[tpos], "forget"))
6705                 return instr_forget_translate(p,
6706                                               action,
6707                                               &tokens[tpos],
6708                                               n_tokens - tpos,
6709                                               instr,
6710                                               data);
6711
6712         if (!strcmp(tokens[tpos], "extern"))
6713                 return instr_extern_translate(p,
6714                                               action,
6715                                               &tokens[tpos],
6716                                               n_tokens - tpos,
6717                                               instr,
6718                                               data);
6719
6720         if (!strcmp(tokens[tpos], "jmp"))
6721                 return instr_jmp_translate(p,
6722                                            action,
6723                                            &tokens[tpos],
6724                                            n_tokens - tpos,
6725                                            instr,
6726                                            data);
6727
6728         if (!strcmp(tokens[tpos], "jmpv"))
6729                 return instr_jmp_valid_translate(p,
6730                                                  action,
6731                                                  &tokens[tpos],
6732                                                  n_tokens - tpos,
6733                                                  instr,
6734                                                  data);
6735
6736         if (!strcmp(tokens[tpos], "jmpnv"))
6737                 return instr_jmp_invalid_translate(p,
6738                                                    action,
6739                                                    &tokens[tpos],
6740                                                    n_tokens - tpos,
6741                                                    instr,
6742                                                    data);
6743
6744         if (!strcmp(tokens[tpos], "jmph"))
6745                 return instr_jmp_hit_translate(p,
6746                                                action,
6747                                                &tokens[tpos],
6748                                                n_tokens - tpos,
6749                                                instr,
6750                                                data);
6751
6752         if (!strcmp(tokens[tpos], "jmpnh"))
6753                 return instr_jmp_miss_translate(p,
6754                                                 action,
6755                                                 &tokens[tpos],
6756                                                 n_tokens - tpos,
6757                                                 instr,
6758                                                 data);
6759
6760         if (!strcmp(tokens[tpos], "jmpa"))
6761                 return instr_jmp_action_hit_translate(p,
6762                                                       action,
6763                                                       &tokens[tpos],
6764                                                       n_tokens - tpos,
6765                                                       instr,
6766                                                       data);
6767
6768         if (!strcmp(tokens[tpos], "jmpna"))
6769                 return instr_jmp_action_miss_translate(p,
6770                                                        action,
6771                                                        &tokens[tpos],
6772                                                        n_tokens - tpos,
6773                                                        instr,
6774                                                        data);
6775
6776         if (!strcmp(tokens[tpos], "jmpeq"))
6777                 return instr_jmp_eq_translate(p,
6778                                               action,
6779                                               &tokens[tpos],
6780                                               n_tokens - tpos,
6781                                               instr,
6782                                               data);
6783
6784         if (!strcmp(tokens[tpos], "jmpneq"))
6785                 return instr_jmp_neq_translate(p,
6786                                                action,
6787                                                &tokens[tpos],
6788                                                n_tokens - tpos,
6789                                                instr,
6790                                                data);
6791
6792         if (!strcmp(tokens[tpos], "jmplt"))
6793                 return instr_jmp_lt_translate(p,
6794                                               action,
6795                                               &tokens[tpos],
6796                                               n_tokens - tpos,
6797                                               instr,
6798                                               data);
6799
6800         if (!strcmp(tokens[tpos], "jmpgt"))
6801                 return instr_jmp_gt_translate(p,
6802                                               action,
6803                                               &tokens[tpos],
6804                                               n_tokens - tpos,
6805                                               instr,
6806                                               data);
6807
6808         if (!strcmp(tokens[tpos], "return"))
6809                 return instr_return_translate(p,
6810                                               action,
6811                                               &tokens[tpos],
6812                                               n_tokens - tpos,
6813                                               instr,
6814                                               data);
6815
6816         CHECK(0, EINVAL);
6817 }
6818
6819 static struct instruction_data *
6820 label_find(struct instruction_data *data, uint32_t n, const char *label)
6821 {
6822         uint32_t i;
6823
6824         for (i = 0; i < n; i++)
6825                 if (!strcmp(label, data[i].label))
6826                         return &data[i];
6827
6828         return NULL;
6829 }
6830
6831 static uint32_t
6832 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6833 {
6834         uint32_t count = 0, i;
6835
6836         if (!label[0])
6837                 return 0;
6838
6839         for (i = 0; i < n; i++)
6840                 if (!strcmp(label, data[i].jmp_label))
6841                         count++;
6842
6843         return count;
6844 }
6845
6846 static int
6847 instr_label_check(struct instruction_data *instruction_data,
6848                   uint32_t n_instructions)
6849 {
6850         uint32_t i;
6851
6852         /* Check that all instruction labels are unique. */
6853         for (i = 0; i < n_instructions; i++) {
6854                 struct instruction_data *data = &instruction_data[i];
6855                 char *label = data->label;
6856                 uint32_t j;
6857
6858                 if (!label[0])
6859                         continue;
6860
6861                 for (j = i + 1; j < n_instructions; j++)
6862                         CHECK(strcmp(label, data[j].label), EINVAL);
6863         }
6864
6865         /* Get users for each instruction label. */
6866         for (i = 0; i < n_instructions; i++) {
6867                 struct instruction_data *data = &instruction_data[i];
6868                 char *label = data->label;
6869
6870                 data->n_users = label_is_used(instruction_data,
6871                                               n_instructions,
6872                                               label);
6873         }
6874
6875         return 0;
6876 }
6877
6878 static int
6879 instr_jmp_resolve(struct instruction *instructions,
6880                   struct instruction_data *instruction_data,
6881                   uint32_t n_instructions)
6882 {
6883         uint32_t i;
6884
6885         for (i = 0; i < n_instructions; i++) {
6886                 struct instruction *instr = &instructions[i];
6887                 struct instruction_data *data = &instruction_data[i];
6888                 struct instruction_data *found;
6889
6890                 if (!instruction_is_jmp(instr))
6891                         continue;
6892
6893                 found = label_find(instruction_data,
6894                                    n_instructions,
6895                                    data->jmp_label);
6896                 CHECK(found, EINVAL);
6897
6898                 instr->jmp.ip = &instructions[found - instruction_data];
6899         }
6900
6901         return 0;
6902 }
6903
6904 static int
6905 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6906              struct action *a,
6907              struct instruction *instr,
6908              struct instruction_data *data __rte_unused,
6909              uint32_t n_instructions)
6910 {
6911         if (!a) {
6912                 enum instruction_type type;
6913                 uint32_t i;
6914
6915                 /* Check that the first instruction is rx. */
6916                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6917
6918                 /* Check that there is at least one tx instruction. */
6919                 for (i = 0; i < n_instructions; i++) {
6920                         type = instr[i].type;
6921
6922                         if (instruction_is_tx(type))
6923                                 break;
6924                 }
6925                 CHECK(i < n_instructions, EINVAL);
6926
6927                 /* Check that the last instruction is either tx or unconditional
6928                  * jump.
6929                  */
6930                 type = instr[n_instructions - 1].type;
6931                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6932         }
6933
6934         if (a) {
6935                 enum instruction_type type;
6936                 uint32_t i;
6937
6938                 /* Check that there is at least one return or tx instruction. */
6939                 for (i = 0; i < n_instructions; i++) {
6940                         type = instr[i].type;
6941
6942                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6943                                 break;
6944                 }
6945                 CHECK(i < n_instructions, EINVAL);
6946         }
6947
6948         return 0;
6949 }
6950
6951 static uint32_t
6952 instr_compact(struct instruction *instructions,
6953               struct instruction_data *instruction_data,
6954               uint32_t n_instructions)
6955 {
6956         uint32_t i, pos = 0;
6957
6958         /* Eliminate the invalid instructions that have been optimized out. */
6959         for (i = 0; i < n_instructions; i++) {
6960                 struct instruction *instr = &instructions[i];
6961                 struct instruction_data *data = &instruction_data[i];
6962
6963                 if (data->invalid)
6964                         continue;
6965
6966                 if (i != pos) {
6967                         memcpy(&instructions[pos], instr, sizeof(*instr));
6968                         memcpy(&instruction_data[pos], data, sizeof(*data));
6969                 }
6970
6971                 pos++;
6972         }
6973
6974         return pos;
6975 }
6976
6977 static int
6978 instr_pattern_extract_many_search(struct instruction *instr,
6979                                   struct instruction_data *data,
6980                                   uint32_t n_instr,
6981                                   uint32_t *n_pattern_instr)
6982 {
6983         uint32_t i;
6984
6985         for (i = 0; i < n_instr; i++) {
6986                 if (data[i].invalid)
6987                         break;
6988
6989                 if (instr[i].type != INSTR_HDR_EXTRACT)
6990                         break;
6991
6992                 if (i == RTE_DIM(instr->io.hdr.header_id))
6993                         break;
6994
6995                 if (i && data[i].n_users)
6996                         break;
6997         }
6998
6999         if (i < 2)
7000                 return 0;
7001
7002         *n_pattern_instr = i;
7003         return 1;
7004 }
7005
7006 static void
7007 instr_pattern_extract_many_replace(struct instruction *instr,
7008                                    struct instruction_data *data,
7009                                    uint32_t n_instr)
7010 {
7011         uint32_t i;
7012
7013         for (i = 1; i < n_instr; i++) {
7014                 instr[0].type++;
7015                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7016                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7017                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7018
7019                 data[i].invalid = 1;
7020         }
7021 }
7022
7023 static uint32_t
7024 instr_pattern_extract_many_optimize(struct instruction *instructions,
7025                                     struct instruction_data *instruction_data,
7026                                     uint32_t n_instructions)
7027 {
7028         uint32_t i;
7029
7030         for (i = 0; i < n_instructions; ) {
7031                 struct instruction *instr = &instructions[i];
7032                 struct instruction_data *data = &instruction_data[i];
7033                 uint32_t n_instr = 0;
7034                 int detected;
7035
7036                 /* Extract many. */
7037                 detected = instr_pattern_extract_many_search(instr,
7038                                                              data,
7039                                                              n_instructions - i,
7040                                                              &n_instr);
7041                 if (detected) {
7042                         instr_pattern_extract_many_replace(instr,
7043                                                            data,
7044                                                            n_instr);
7045                         i += n_instr;
7046                         continue;
7047                 }
7048
7049                 /* No pattern starting at the current instruction. */
7050                 i++;
7051         }
7052
7053         /* Eliminate the invalid instructions that have been optimized out. */
7054         n_instructions = instr_compact(instructions,
7055                                        instruction_data,
7056                                        n_instructions);
7057
7058         return n_instructions;
7059 }
7060
7061 static int
7062 instr_pattern_emit_many_tx_search(struct instruction *instr,
7063                                   struct instruction_data *data,
7064                                   uint32_t n_instr,
7065                                   uint32_t *n_pattern_instr)
7066 {
7067         uint32_t i;
7068
7069         for (i = 0; i < n_instr; i++) {
7070                 if (data[i].invalid)
7071                         break;
7072
7073                 if (instr[i].type != INSTR_HDR_EMIT)
7074                         break;
7075
7076                 if (i == RTE_DIM(instr->io.hdr.header_id))
7077                         break;
7078
7079                 if (i && data[i].n_users)
7080                         break;
7081         }
7082
7083         if (!i)
7084                 return 0;
7085
7086         if (!instruction_is_tx(instr[i].type))
7087                 return 0;
7088
7089         if (data[i].n_users)
7090                 return 0;
7091
7092         i++;
7093
7094         *n_pattern_instr = i;
7095         return 1;
7096 }
7097
7098 static void
7099 instr_pattern_emit_many_tx_replace(struct instruction *instr,
7100                                    struct instruction_data *data,
7101                                    uint32_t n_instr)
7102 {
7103         uint32_t i;
7104
7105         /* Any emit instruction in addition to the first one. */
7106         for (i = 1; i < n_instr - 1; i++) {
7107                 instr[0].type++;
7108                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7109                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7110                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7111
7112                 data[i].invalid = 1;
7113         }
7114
7115         /* The TX instruction is the last one in the pattern. */
7116         instr[0].type++;
7117         instr[0].io.io.offset = instr[i].io.io.offset;
7118         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7119         data[i].invalid = 1;
7120 }
7121
7122 static uint32_t
7123 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
7124                                     struct instruction_data *instruction_data,
7125                                     uint32_t n_instructions)
7126 {
7127         uint32_t i;
7128
7129         for (i = 0; i < n_instructions; ) {
7130                 struct instruction *instr = &instructions[i];
7131                 struct instruction_data *data = &instruction_data[i];
7132                 uint32_t n_instr = 0;
7133                 int detected;
7134
7135                 /* Emit many + TX. */
7136                 detected = instr_pattern_emit_many_tx_search(instr,
7137                                                              data,
7138                                                              n_instructions - i,
7139                                                              &n_instr);
7140                 if (detected) {
7141                         instr_pattern_emit_many_tx_replace(instr,
7142                                                            data,
7143                                                            n_instr);
7144                         i += n_instr;
7145                         continue;
7146                 }
7147
7148                 /* No pattern starting at the current instruction. */
7149                 i++;
7150         }
7151
7152         /* Eliminate the invalid instructions that have been optimized out. */
7153         n_instructions = instr_compact(instructions,
7154                                        instruction_data,
7155                                        n_instructions);
7156
7157         return n_instructions;
7158 }
7159
7160 static uint32_t
7161 action_arg_src_mov_count(struct action *a,
7162                          uint32_t arg_id,
7163                          struct instruction *instructions,
7164                          struct instruction_data *instruction_data,
7165                          uint32_t n_instructions);
7166
7167 static int
7168 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
7169                                       struct action *a,
7170                                       struct instruction *instr,
7171                                       struct instruction_data *data,
7172                                       uint32_t n_instr,
7173                                       struct instruction *instructions,
7174                                       struct instruction_data *instruction_data,
7175                                       uint32_t n_instructions,
7176                                       uint32_t *n_pattern_instr)
7177 {
7178         struct header *h;
7179         uint32_t src_field_id, i, j;
7180
7181         /* Prerequisites. */
7182         if (!a || !a->st)
7183                 return 0;
7184
7185         /* First instruction: MOV_HM. */
7186         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
7187                 return 0;
7188
7189         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7190         if (!h || h->st->var_size)
7191                 return 0;
7192
7193         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7194                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7195                         break;
7196
7197         if (src_field_id == a->st->n_fields)
7198                 return 0;
7199
7200         if (instr[0].mov.dst.offset ||
7201             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
7202             instr[0].mov.src.struct_id ||
7203             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
7204             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
7205                 return 0;
7206
7207         if ((n_instr < h->st->n_fields + 1) ||
7208              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
7209                 return 0;
7210
7211         /* Subsequent instructions: MOV_HM. */
7212         for (i = 1; i < h->st->n_fields; i++)
7213                 if (data[i].invalid ||
7214                     data[i].n_users ||
7215                     (instr[i].type != INSTR_MOV_HM) ||
7216                     (instr[i].mov.dst.struct_id != h->struct_id) ||
7217                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
7218                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
7219                     instr[i].mov.src.struct_id ||
7220                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
7221                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
7222                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
7223                         return 0;
7224
7225         /* Last instruction: HDR_VALIDATE. */
7226         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
7227             (instr[i].valid.header_id != h->id))
7228                 return 0;
7229
7230         /* Check that none of the action args that are used as source for this
7231          * DMA transfer are not used as source in any other mov instruction.
7232          */
7233         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
7234                 uint32_t n_users;
7235
7236                 n_users = action_arg_src_mov_count(a,
7237                                                    j,
7238                                                    instructions,
7239                                                    instruction_data,
7240                                                    n_instructions);
7241                 if (n_users > 1)
7242                         return 0;
7243         }
7244
7245         *n_pattern_instr = 1 + i;
7246         return 1;
7247 }
7248
7249 static void
7250 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
7251                                        struct action *a,
7252                                        struct instruction *instr,
7253                                        struct instruction_data *data,
7254                                        uint32_t n_instr)
7255 {
7256         struct header *h;
7257         uint32_t src_field_id, src_offset, i;
7258
7259         /* Read from the instructions before they are modified. */
7260         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
7261         if (!h)
7262                 return;
7263
7264         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
7265                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
7266                         break;
7267
7268         if (src_field_id == a->st->n_fields)
7269                 return;
7270
7271         src_offset = instr[0].mov.src.offset;
7272
7273         /* Modify the instructions. */
7274         instr[0].type = INSTR_DMA_HT;
7275         instr[0].dma.dst.header_id[0] = h->id;
7276         instr[0].dma.dst.struct_id[0] = h->struct_id;
7277         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
7278         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
7279
7280         for (i = 1; i < n_instr; i++)
7281                 data[i].invalid = 1;
7282
7283         /* Update the endianness of the action arguments to header endianness. */
7284         for (i = 0; i < h->st->n_fields; i++)
7285                 a->args_endianness[src_field_id + i] = 1;
7286 }
7287
7288 static uint32_t
7289 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
7290                                         struct action *a,
7291                                         struct instruction *instructions,
7292                                         struct instruction_data *instruction_data,
7293                                         uint32_t n_instructions)
7294 {
7295         uint32_t i;
7296
7297         if (!a || !a->st)
7298                 return n_instructions;
7299
7300         for (i = 0; i < n_instructions; ) {
7301                 struct instruction *instr = &instructions[i];
7302                 struct instruction_data *data = &instruction_data[i];
7303                 uint32_t n_instr = 0;
7304                 int detected;
7305
7306                 /* Mov all + validate. */
7307                 detected = instr_pattern_mov_all_validate_search(p,
7308                                                                  a,
7309                                                                  instr,
7310                                                                  data,
7311                                                                  n_instructions - i,
7312                                                                  instructions,
7313                                                                  instruction_data,
7314                                                                  n_instructions,
7315                                                                  &n_instr);
7316                 if (detected) {
7317                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
7318                         i += n_instr;
7319                         continue;
7320                 }
7321
7322                 /* No pattern starting at the current instruction. */
7323                 i++;
7324         }
7325
7326         /* Eliminate the invalid instructions that have been optimized out. */
7327         n_instructions = instr_compact(instructions,
7328                                        instruction_data,
7329                                        n_instructions);
7330
7331         return n_instructions;
7332 }
7333
7334 static int
7335 instr_pattern_dma_many_search(struct instruction *instr,
7336                               struct instruction_data *data,
7337                               uint32_t n_instr,
7338                               uint32_t *n_pattern_instr)
7339 {
7340         uint32_t i;
7341
7342         for (i = 0; i < n_instr; i++) {
7343                 if (data[i].invalid)
7344                         break;
7345
7346                 if (instr[i].type != INSTR_DMA_HT)
7347                         break;
7348
7349                 if (i == RTE_DIM(instr->dma.dst.header_id))
7350                         break;
7351
7352                 if (i && data[i].n_users)
7353                         break;
7354         }
7355
7356         if (i < 2)
7357                 return 0;
7358
7359         *n_pattern_instr = i;
7360         return 1;
7361 }
7362
7363 static void
7364 instr_pattern_dma_many_replace(struct instruction *instr,
7365                                struct instruction_data *data,
7366                                uint32_t n_instr)
7367 {
7368         uint32_t i;
7369
7370         for (i = 1; i < n_instr; i++) {
7371                 instr[0].type++;
7372                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7373                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7374                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7375                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7376
7377                 data[i].invalid = 1;
7378         }
7379 }
7380
7381 static uint32_t
7382 instr_pattern_dma_many_optimize(struct instruction *instructions,
7383                struct instruction_data *instruction_data,
7384                uint32_t n_instructions)
7385 {
7386         uint32_t i;
7387
7388         for (i = 0; i < n_instructions; ) {
7389                 struct instruction *instr = &instructions[i];
7390                 struct instruction_data *data = &instruction_data[i];
7391                 uint32_t n_instr = 0;
7392                 int detected;
7393
7394                 /* DMA many. */
7395                 detected = instr_pattern_dma_many_search(instr,
7396                                                          data,
7397                                                          n_instructions - i,
7398                                                          &n_instr);
7399                 if (detected) {
7400                         instr_pattern_dma_many_replace(instr, data, n_instr);
7401                         i += n_instr;
7402                         continue;
7403                 }
7404
7405                 /* No pattern starting at the current instruction. */
7406                 i++;
7407         }
7408
7409         /* Eliminate the invalid instructions that have been optimized out. */
7410         n_instructions = instr_compact(instructions,
7411                                        instruction_data,
7412                                        n_instructions);
7413
7414         return n_instructions;
7415 }
7416
7417 static uint32_t
7418 instr_optimize(struct rte_swx_pipeline *p,
7419                struct action *a,
7420                struct instruction *instructions,
7421                struct instruction_data *instruction_data,
7422                uint32_t n_instructions)
7423 {
7424         /* Extract many. */
7425         n_instructions = instr_pattern_extract_many_optimize(instructions,
7426                                                              instruction_data,
7427                                                              n_instructions);
7428
7429         /* Emit many + TX. */
7430         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7431                                                              instruction_data,
7432                                                              n_instructions);
7433
7434         /* Mov all + validate. */
7435         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7436                                                                  a,
7437                                                                  instructions,
7438                                                                  instruction_data,
7439                                                                  n_instructions);
7440
7441         /* DMA many. */
7442         n_instructions = instr_pattern_dma_many_optimize(instructions,
7443                                                          instruction_data,
7444                                                          n_instructions);
7445
7446         return n_instructions;
7447 }
7448
7449 static int
7450 instruction_config(struct rte_swx_pipeline *p,
7451                    struct action *a,
7452                    const char **instructions,
7453                    uint32_t n_instructions)
7454 {
7455         struct instruction *instr = NULL;
7456         struct instruction_data *data = NULL;
7457         int err = 0;
7458         uint32_t i;
7459
7460         CHECK(n_instructions, EINVAL);
7461         CHECK(instructions, EINVAL);
7462         for (i = 0; i < n_instructions; i++)
7463                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7464
7465         /* Memory allocation. */
7466         instr = calloc(n_instructions, sizeof(struct instruction));
7467         if (!instr) {
7468                 err = -ENOMEM;
7469                 goto error;
7470         }
7471
7472         data = calloc(n_instructions, sizeof(struct instruction_data));
7473         if (!data) {
7474                 err = -ENOMEM;
7475                 goto error;
7476         }
7477
7478         for (i = 0; i < n_instructions; i++) {
7479                 char *string = strdup(instructions[i]);
7480                 if (!string) {
7481                         err = -ENOMEM;
7482                         goto error;
7483                 }
7484
7485                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7486                 if (err) {
7487                         free(string);
7488                         goto error;
7489                 }
7490
7491                 free(string);
7492         }
7493
7494         err = instr_label_check(data, n_instructions);
7495         if (err)
7496                 goto error;
7497
7498         err = instr_verify(p, a, instr, data, n_instructions);
7499         if (err)
7500                 goto error;
7501
7502         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7503
7504         err = instr_jmp_resolve(instr, data, n_instructions);
7505         if (err)
7506                 goto error;
7507
7508         if (a) {
7509                 a->instructions = instr;
7510                 a->n_instructions = n_instructions;
7511         } else {
7512                 p->instructions = instr;
7513                 p->n_instructions = n_instructions;
7514         }
7515
7516         free(data);
7517         return 0;
7518
7519 error:
7520         free(data);
7521         free(instr);
7522         return err;
7523 }
7524
7525 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
7526
7527 static instr_exec_t instruction_table[] = {
7528         [INSTR_RX] = instr_rx_exec,
7529         [INSTR_TX] = instr_tx_exec,
7530         [INSTR_TX_I] = instr_tx_i_exec,
7531
7532         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7533         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7534         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7535         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7536         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7537         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7538         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7539         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7540         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7541         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7542
7543         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7544         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7545         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7546         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7547         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7548         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7549         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7550         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7551         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7552
7553         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7554         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7555
7556         [INSTR_MOV] = instr_mov_exec,
7557         [INSTR_MOV_MH] = instr_mov_mh_exec,
7558         [INSTR_MOV_HM] = instr_mov_hm_exec,
7559         [INSTR_MOV_HH] = instr_mov_hh_exec,
7560         [INSTR_MOV_I] = instr_mov_i_exec,
7561
7562         [INSTR_DMA_HT] = instr_dma_ht_exec,
7563         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7564         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7565         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7566         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7567         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7568         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7569         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7570
7571         [INSTR_ALU_ADD] = instr_alu_add_exec,
7572         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7573         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7574         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7575         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7576         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7577
7578         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7579         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7580         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7581         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7582         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7583         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7584
7585         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7586         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7587         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7588         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7589
7590         [INSTR_ALU_AND] = instr_alu_and_exec,
7591         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7592         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7593         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7594         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7595
7596         [INSTR_ALU_OR] = instr_alu_or_exec,
7597         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7598         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7599         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7600         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7601
7602         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7603         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7604         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7605         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7606         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7607
7608         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7609         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7610         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7611         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7612         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7613         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7614
7615         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7616         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7617         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7618         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7619         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7620         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7621
7622         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7623         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7624         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7625
7626         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7627         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7628         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7629         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7630         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7631         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7632
7633         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7634         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7635         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7636         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7637         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7638         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7639         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7640         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7641         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7642
7643         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7644         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7645         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7646         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7647         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7648         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7649         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7650         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7651         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7652
7653         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7654         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7655         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7656
7657         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7658         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7659         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7660         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7661         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7662         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7663         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7664         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7665         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7666         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7667         [INSTR_METER_IMM] = instr_meter_imm_exec,
7668         [INSTR_METER_IMI] = instr_meter_imi_exec,
7669
7670         [INSTR_TABLE] = instr_table_exec,
7671         [INSTR_SELECTOR] = instr_selector_exec,
7672         [INSTR_LEARNER] = instr_learner_exec,
7673         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7674         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7675         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7676         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7677
7678         [INSTR_JMP] = instr_jmp_exec,
7679         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7680         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7681         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7682         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7683         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7684         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7685
7686         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7687         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7688         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7689         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7690         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7691
7692         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7693         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7694         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7695         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7696         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7697
7698         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7699         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7700         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7701         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7702         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7703         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7704
7705         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7706         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7707         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7708         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7709         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7710         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7711
7712         [INSTR_RETURN] = instr_return_exec,
7713 };
7714
7715 static inline void
7716 instr_exec(struct rte_swx_pipeline *p)
7717 {
7718         struct thread *t = &p->threads[p->thread_id];
7719         struct instruction *ip = t->ip;
7720         instr_exec_t instr = instruction_table[ip->type];
7721
7722         instr(p);
7723 }
7724
7725 /*
7726  * Action.
7727  */
7728 static struct action *
7729 action_find(struct rte_swx_pipeline *p, const char *name)
7730 {
7731         struct action *elem;
7732
7733         if (!name)
7734                 return NULL;
7735
7736         TAILQ_FOREACH(elem, &p->actions, node)
7737                 if (strcmp(elem->name, name) == 0)
7738                         return elem;
7739
7740         return NULL;
7741 }
7742
7743 static struct action *
7744 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7745 {
7746         struct action *action = NULL;
7747
7748         TAILQ_FOREACH(action, &p->actions, node)
7749                 if (action->id == id)
7750                         return action;
7751
7752         return NULL;
7753 }
7754
7755 static struct field *
7756 action_field_find(struct action *a, const char *name)
7757 {
7758         return a->st ? struct_type_field_find(a->st, name) : NULL;
7759 }
7760
7761 static struct field *
7762 action_field_parse(struct action *action, const char *name)
7763 {
7764         if (name[0] != 't' || name[1] != '.')
7765                 return NULL;
7766
7767         return action_field_find(action, &name[2]);
7768 }
7769
7770 static int
7771 action_has_nbo_args(struct action *a)
7772 {
7773         uint32_t i;
7774
7775         /* Return if the action does not have any args. */
7776         if (!a->st)
7777                 return 0; /* FALSE */
7778
7779         for (i = 0; i < a->st->n_fields; i++)
7780                 if (a->args_endianness[i])
7781                         return 1; /* TRUE */
7782
7783         return 0; /* FALSE */
7784 }
7785
7786 static int
7787 action_does_learning(struct action *a)
7788 {
7789         uint32_t i;
7790
7791         for (i = 0; i < a->n_instructions; i++)
7792                 switch (a->instructions[i].type) {
7793                 case INSTR_LEARNER_LEARN:
7794                         return 1; /* TRUE */
7795
7796                 case INSTR_LEARNER_FORGET:
7797                         return 1; /* TRUE */
7798
7799                 default:
7800                         continue;
7801                 }
7802
7803         return 0; /* FALSE */
7804 }
7805
7806 int
7807 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7808                                const char *name,
7809                                const char *args_struct_type_name,
7810                                const char **instructions,
7811                                uint32_t n_instructions)
7812 {
7813         struct struct_type *args_struct_type = NULL;
7814         struct action *a;
7815         int err;
7816
7817         CHECK(p, EINVAL);
7818
7819         CHECK_NAME(name, EINVAL);
7820         CHECK(!action_find(p, name), EEXIST);
7821
7822         if (args_struct_type_name) {
7823                 CHECK_NAME(args_struct_type_name, EINVAL);
7824                 args_struct_type = struct_type_find(p, args_struct_type_name);
7825                 CHECK(args_struct_type, EINVAL);
7826                 CHECK(!args_struct_type->var_size, EINVAL);
7827         }
7828
7829         /* Node allocation. */
7830         a = calloc(1, sizeof(struct action));
7831         CHECK(a, ENOMEM);
7832         if (args_struct_type) {
7833                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7834                 if (!a->args_endianness) {
7835                         free(a);
7836                         CHECK(0, ENOMEM);
7837                 }
7838         }
7839
7840         /* Node initialization. */
7841         strcpy(a->name, name);
7842         a->st = args_struct_type;
7843         a->id = p->n_actions;
7844
7845         /* Instruction translation. */
7846         err = instruction_config(p, a, instructions, n_instructions);
7847         if (err) {
7848                 free(a->args_endianness);
7849                 free(a);
7850                 return err;
7851         }
7852
7853         /* Node add to tailq. */
7854         TAILQ_INSERT_TAIL(&p->actions, a, node);
7855         p->n_actions++;
7856
7857         return 0;
7858 }
7859
7860 static int
7861 action_build(struct rte_swx_pipeline *p)
7862 {
7863         struct action *action;
7864
7865         p->action_instructions = calloc(p->n_actions,
7866                                         sizeof(struct instruction *));
7867         CHECK(p->action_instructions, ENOMEM);
7868
7869         TAILQ_FOREACH(action, &p->actions, node)
7870                 p->action_instructions[action->id] = action->instructions;
7871
7872         return 0;
7873 }
7874
7875 static void
7876 action_build_free(struct rte_swx_pipeline *p)
7877 {
7878         free(p->action_instructions);
7879         p->action_instructions = NULL;
7880 }
7881
7882 static void
7883 action_free(struct rte_swx_pipeline *p)
7884 {
7885         action_build_free(p);
7886
7887         for ( ; ; ) {
7888                 struct action *action;
7889
7890                 action = TAILQ_FIRST(&p->actions);
7891                 if (!action)
7892                         break;
7893
7894                 TAILQ_REMOVE(&p->actions, action, node);
7895                 free(action->instructions);
7896                 free(action);
7897         }
7898 }
7899
7900 static uint32_t
7901 action_arg_src_mov_count(struct action *a,
7902                          uint32_t arg_id,
7903                          struct instruction *instructions,
7904                          struct instruction_data *instruction_data,
7905                          uint32_t n_instructions)
7906 {
7907         uint32_t offset, n_users = 0, i;
7908
7909         if (!a->st ||
7910             (arg_id >= a->st->n_fields) ||
7911             !instructions ||
7912             !instruction_data ||
7913             !n_instructions)
7914                 return 0;
7915
7916         offset = a->st->fields[arg_id].offset / 8;
7917
7918         for (i = 0; i < n_instructions; i++) {
7919                 struct instruction *instr = &instructions[i];
7920                 struct instruction_data *data = &instruction_data[i];
7921
7922                 if (data->invalid ||
7923                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7924                     instr->mov.src.struct_id ||
7925                     (instr->mov.src.offset != offset))
7926                         continue;
7927
7928                 n_users++;
7929         }
7930
7931         return n_users;
7932 }
7933
7934 /*
7935  * Table.
7936  */
7937 static struct table_type *
7938 table_type_find(struct rte_swx_pipeline *p, const char *name)
7939 {
7940         struct table_type *elem;
7941
7942         TAILQ_FOREACH(elem, &p->table_types, node)
7943                 if (strcmp(elem->name, name) == 0)
7944                         return elem;
7945
7946         return NULL;
7947 }
7948
7949 static struct table_type *
7950 table_type_resolve(struct rte_swx_pipeline *p,
7951                    const char *recommended_type_name,
7952                    enum rte_swx_table_match_type match_type)
7953 {
7954         struct table_type *elem;
7955
7956         /* Only consider the recommended type if the match type is correct. */
7957         if (recommended_type_name)
7958                 TAILQ_FOREACH(elem, &p->table_types, node)
7959                         if (!strcmp(elem->name, recommended_type_name) &&
7960                             (elem->match_type == match_type))
7961                                 return elem;
7962
7963         /* Ignore the recommended type and get the first element with this match
7964          * type.
7965          */
7966         TAILQ_FOREACH(elem, &p->table_types, node)
7967                 if (elem->match_type == match_type)
7968                         return elem;
7969
7970         return NULL;
7971 }
7972
7973 static struct table *
7974 table_find(struct rte_swx_pipeline *p, const char *name)
7975 {
7976         struct table *elem;
7977
7978         TAILQ_FOREACH(elem, &p->tables, node)
7979                 if (strcmp(elem->name, name) == 0)
7980                         return elem;
7981
7982         return NULL;
7983 }
7984
7985 static struct table *
7986 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7987 {
7988         struct table *table = NULL;
7989
7990         TAILQ_FOREACH(table, &p->tables, node)
7991                 if (table->id == id)
7992                         return table;
7993
7994         return NULL;
7995 }
7996
7997 int
7998 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7999                                      const char *name,
8000                                      enum rte_swx_table_match_type match_type,
8001                                      struct rte_swx_table_ops *ops)
8002 {
8003         struct table_type *elem;
8004
8005         CHECK(p, EINVAL);
8006
8007         CHECK_NAME(name, EINVAL);
8008         CHECK(!table_type_find(p, name), EEXIST);
8009
8010         CHECK(ops, EINVAL);
8011         CHECK(ops->create, EINVAL);
8012         CHECK(ops->lkp, EINVAL);
8013         CHECK(ops->free, EINVAL);
8014
8015         /* Node allocation. */
8016         elem = calloc(1, sizeof(struct table_type));
8017         CHECK(elem, ENOMEM);
8018
8019         /* Node initialization. */
8020         strcpy(elem->name, name);
8021         elem->match_type = match_type;
8022         memcpy(&elem->ops, ops, sizeof(*ops));
8023
8024         /* Node add to tailq. */
8025         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8026
8027         return 0;
8028 }
8029
8030 static int
8031 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8032                          uint32_t n_fields,
8033                          enum rte_swx_table_match_type *match_type)
8034 {
8035         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
8036
8037         for (i = 0; i < n_fields; i++) {
8038                 struct rte_swx_match_field_params  *f = &fields[i];
8039
8040                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
8041                         n_fields_em++;
8042
8043                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
8044                         n_fields_lpm++;
8045         }
8046
8047         if ((n_fields_lpm > 1) ||
8048             (n_fields_lpm && (n_fields_em != n_fields - 1)))
8049                 return -EINVAL;
8050
8051         *match_type = (n_fields_em == n_fields) ?
8052                        RTE_SWX_TABLE_MATCH_EXACT :
8053                        RTE_SWX_TABLE_MATCH_WILDCARD;
8054
8055         return 0;
8056 }
8057
8058 static int
8059 table_match_fields_check(struct rte_swx_pipeline *p,
8060                          struct rte_swx_pipeline_table_params *params,
8061                          struct header **header)
8062 {
8063         struct header *h0 = NULL;
8064         struct field *hf, *mf;
8065         uint32_t *offset = NULL, i;
8066         int status = 0;
8067
8068         /* Return if no match fields. */
8069         if (!params->n_fields) {
8070                 if (params->fields) {
8071                         status = -EINVAL;
8072                         goto end;
8073                 }
8074
8075                 if (header)
8076                         *header = NULL;
8077
8078                 return 0;
8079         }
8080
8081         /* Memory allocation. */
8082         offset = calloc(params->n_fields, sizeof(uint32_t));
8083         if (!offset) {
8084                 status = -ENOMEM;
8085                 goto end;
8086         }
8087
8088         /* Check that all the match fields belong to either the same header or
8089          * to the meta-data.
8090          */
8091         hf = header_field_parse(p, params->fields[0].name, &h0);
8092         mf = metadata_field_parse(p, params->fields[0].name);
8093         if ((!hf && !mf) || (hf && hf->var_size)) {
8094                 status = -EINVAL;
8095                 goto end;
8096         }
8097
8098         offset[0] = h0 ? hf->offset : mf->offset;
8099
8100         for (i = 1; i < params->n_fields; i++)
8101                 if (h0) {
8102                         struct header *h;
8103
8104                         hf = header_field_parse(p, params->fields[i].name, &h);
8105                         if (!hf || (h->id != h0->id) || hf->var_size) {
8106                                 status = -EINVAL;
8107                                 goto end;
8108                         }
8109
8110                         offset[i] = hf->offset;
8111                 } else {
8112                         mf = metadata_field_parse(p, params->fields[i].name);
8113                         if (!mf) {
8114                                 status = -EINVAL;
8115                                 goto end;
8116                         }
8117
8118                         offset[i] = mf->offset;
8119                 }
8120
8121         /* Check that there are no duplicated match fields. */
8122         for (i = 0; i < params->n_fields; i++) {
8123                 uint32_t j;
8124
8125                 for (j = 0; j < i; j++)
8126                         if (offset[j] == offset[i]) {
8127                                 status = -EINVAL;
8128                                 goto end;
8129                         }
8130         }
8131
8132         /* Return. */
8133         if (header)
8134                 *header = h0;
8135
8136 end:
8137         free(offset);
8138         return status;
8139 }
8140
8141 int
8142 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8143                               const char *name,
8144                               struct rte_swx_pipeline_table_params *params,
8145                               const char *recommended_table_type_name,
8146                               const char *args,
8147                               uint32_t size)
8148 {
8149         struct table_type *type;
8150         struct table *t;
8151         struct action *default_action;
8152         struct header *header = NULL;
8153         uint32_t action_data_size_max = 0, i;
8154         int status = 0;
8155
8156         CHECK(p, EINVAL);
8157
8158         CHECK_NAME(name, EINVAL);
8159         CHECK(!table_find(p, name), EEXIST);
8160         CHECK(!selector_find(p, name), EEXIST);
8161         CHECK(!learner_find(p, name), EEXIST);
8162
8163         CHECK(params, EINVAL);
8164
8165         /* Match checks. */
8166         status = table_match_fields_check(p, params, &header);
8167         if (status)
8168                 return status;
8169
8170         /* Action checks. */
8171         CHECK(params->n_actions, EINVAL);
8172         CHECK(params->action_names, EINVAL);
8173         for (i = 0; i < params->n_actions; i++) {
8174                 const char *action_name = params->action_names[i];
8175                 struct action *a;
8176                 uint32_t action_data_size;
8177
8178                 CHECK_NAME(action_name, EINVAL);
8179
8180                 a = action_find(p, action_name);
8181                 CHECK(a, EINVAL);
8182                 CHECK(!action_does_learning(a), EINVAL);
8183
8184                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8185                 if (action_data_size > action_data_size_max)
8186                         action_data_size_max = action_data_size;
8187         }
8188
8189         CHECK_NAME(params->default_action_name, EINVAL);
8190         for (i = 0; i < p->n_actions; i++)
8191                 if (!strcmp(params->action_names[i],
8192                             params->default_action_name))
8193                         break;
8194         CHECK(i < params->n_actions, EINVAL);
8195         default_action = action_find(p, params->default_action_name);
8196         CHECK((default_action->st && params->default_action_data) ||
8197               !params->default_action_data, EINVAL);
8198
8199         /* Table type checks. */
8200         if (recommended_table_type_name)
8201                 CHECK_NAME(recommended_table_type_name, EINVAL);
8202
8203         if (params->n_fields) {
8204                 enum rte_swx_table_match_type match_type;
8205
8206                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
8207                 if (status)
8208                         return status;
8209
8210                 type = table_type_resolve(p, recommended_table_type_name, match_type);
8211                 CHECK(type, EINVAL);
8212         } else {
8213                 type = NULL;
8214         }
8215
8216         /* Memory allocation. */
8217         t = calloc(1, sizeof(struct table));
8218         CHECK(t, ENOMEM);
8219
8220         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8221         if (!t->fields) {
8222                 free(t);
8223                 CHECK(0, ENOMEM);
8224         }
8225
8226         t->actions = calloc(params->n_actions, sizeof(struct action *));
8227         if (!t->actions) {
8228                 free(t->fields);
8229                 free(t);
8230                 CHECK(0, ENOMEM);
8231         }
8232
8233         if (action_data_size_max) {
8234                 t->default_action_data = calloc(1, action_data_size_max);
8235                 if (!t->default_action_data) {
8236                         free(t->actions);
8237                         free(t->fields);
8238                         free(t);
8239                         CHECK(0, ENOMEM);
8240                 }
8241         }
8242
8243         /* Node initialization. */
8244         strcpy(t->name, name);
8245         if (args && args[0])
8246                 strcpy(t->args, args);
8247         t->type = type;
8248
8249         for (i = 0; i < params->n_fields; i++) {
8250                 struct rte_swx_match_field_params *field = &params->fields[i];
8251                 struct match_field *f = &t->fields[i];
8252
8253                 f->match_type = field->match_type;
8254                 f->field = header ?
8255                         header_field_parse(p, field->name, NULL) :
8256                         metadata_field_parse(p, field->name);
8257         }
8258         t->n_fields = params->n_fields;
8259         t->header = header;
8260
8261         for (i = 0; i < params->n_actions; i++)
8262                 t->actions[i] = action_find(p, params->action_names[i]);
8263         t->default_action = default_action;
8264         if (default_action->st)
8265                 memcpy(t->default_action_data,
8266                        params->default_action_data,
8267                        default_action->st->n_bits / 8);
8268         t->n_actions = params->n_actions;
8269         t->default_action_is_const = params->default_action_is_const;
8270         t->action_data_size_max = action_data_size_max;
8271
8272         t->size = size;
8273         t->id = p->n_tables;
8274
8275         /* Node add to tailq. */
8276         TAILQ_INSERT_TAIL(&p->tables, t, node);
8277         p->n_tables++;
8278
8279         return 0;
8280 }
8281
8282 static struct rte_swx_table_params *
8283 table_params_get(struct table *table)
8284 {
8285         struct rte_swx_table_params *params;
8286         struct field *first, *last;
8287         uint8_t *key_mask;
8288         uint32_t key_size, key_offset, action_data_size, i;
8289
8290         /* Memory allocation. */
8291         params = calloc(1, sizeof(struct rte_swx_table_params));
8292         if (!params)
8293                 return NULL;
8294
8295         /* Find first (smallest offset) and last (biggest offset) match fields. */
8296         first = table->fields[0].field;
8297         last = table->fields[0].field;
8298
8299         for (i = 0; i < table->n_fields; i++) {
8300                 struct field *f = table->fields[i].field;
8301
8302                 if (f->offset < first->offset)
8303                         first = f;
8304
8305                 if (f->offset > last->offset)
8306                         last = f;
8307         }
8308
8309         /* Key offset and size. */
8310         key_offset = first->offset / 8;
8311         key_size = (last->offset + last->n_bits - first->offset) / 8;
8312
8313         /* Memory allocation. */
8314         key_mask = calloc(1, key_size);
8315         if (!key_mask) {
8316                 free(params);
8317                 return NULL;
8318         }
8319
8320         /* Key mask. */
8321         for (i = 0; i < table->n_fields; i++) {
8322                 struct field *f = table->fields[i].field;
8323                 uint32_t start = (f->offset - first->offset) / 8;
8324                 size_t size = f->n_bits / 8;
8325
8326                 memset(&key_mask[start], 0xFF, size);
8327         }
8328
8329         /* Action data size. */
8330         action_data_size = 0;
8331         for (i = 0; i < table->n_actions; i++) {
8332                 struct action *action = table->actions[i];
8333                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8334
8335                 if (ads > action_data_size)
8336                         action_data_size = ads;
8337         }
8338
8339         /* Fill in. */
8340         params->match_type = table->type->match_type;
8341         params->key_size = key_size;
8342         params->key_offset = key_offset;
8343         params->key_mask0 = key_mask;
8344         params->action_data_size = action_data_size;
8345         params->n_keys_max = table->size;
8346
8347         return params;
8348 }
8349
8350 static void
8351 table_params_free(struct rte_swx_table_params *params)
8352 {
8353         if (!params)
8354                 return;
8355
8356         free(params->key_mask0);
8357         free(params);
8358 }
8359
8360 static int
8361 table_stub_lkp(void *table __rte_unused,
8362                void *mailbox __rte_unused,
8363                uint8_t **key __rte_unused,
8364                uint64_t *action_id __rte_unused,
8365                uint8_t **action_data __rte_unused,
8366                int *hit)
8367 {
8368         *hit = 0;
8369         return 1; /* DONE. */
8370 }
8371
8372 static int
8373 table_build(struct rte_swx_pipeline *p)
8374 {
8375         uint32_t i;
8376
8377         /* Per pipeline: table statistics. */
8378         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8379         CHECK(p->table_stats, ENOMEM);
8380
8381         for (i = 0; i < p->n_tables; i++) {
8382                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8383                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8384         }
8385
8386         /* Per thread: table runt-time. */
8387         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8388                 struct thread *t = &p->threads[i];
8389                 struct table *table;
8390
8391                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8392                 CHECK(t->tables, ENOMEM);
8393
8394                 TAILQ_FOREACH(table, &p->tables, node) {
8395                         struct table_runtime *r = &t->tables[table->id];
8396
8397                         if (table->type) {
8398                                 uint64_t size;
8399
8400                                 size = table->type->ops.mailbox_size_get();
8401
8402                                 /* r->func. */
8403                                 r->func = table->type->ops.lkp;
8404
8405                                 /* r->mailbox. */
8406                                 if (size) {
8407                                         r->mailbox = calloc(1, size);
8408                                         CHECK(r->mailbox, ENOMEM);
8409                                 }
8410
8411                                 /* r->key. */
8412                                 r->key = table->header ?
8413                                         &t->structs[table->header->struct_id] :
8414                                         &t->structs[p->metadata_struct_id];
8415                         } else {
8416                                 r->func = table_stub_lkp;
8417                         }
8418                 }
8419         }
8420
8421         return 0;
8422 }
8423
8424 static void
8425 table_build_free(struct rte_swx_pipeline *p)
8426 {
8427         uint32_t i;
8428
8429         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8430                 struct thread *t = &p->threads[i];
8431                 uint32_t j;
8432
8433                 if (!t->tables)
8434                         continue;
8435
8436                 for (j = 0; j < p->n_tables; j++) {
8437                         struct table_runtime *r = &t->tables[j];
8438
8439                         free(r->mailbox);
8440                 }
8441
8442                 free(t->tables);
8443                 t->tables = NULL;
8444         }
8445
8446         if (p->table_stats) {
8447                 for (i = 0; i < p->n_tables; i++)
8448                         free(p->table_stats[i].n_pkts_action);
8449
8450                 free(p->table_stats);
8451         }
8452 }
8453
8454 static void
8455 table_free(struct rte_swx_pipeline *p)
8456 {
8457         table_build_free(p);
8458
8459         /* Tables. */
8460         for ( ; ; ) {
8461                 struct table *elem;
8462
8463                 elem = TAILQ_FIRST(&p->tables);
8464                 if (!elem)
8465                         break;
8466
8467                 TAILQ_REMOVE(&p->tables, elem, node);
8468                 free(elem->fields);
8469                 free(elem->actions);
8470                 free(elem->default_action_data);
8471                 free(elem);
8472         }
8473
8474         /* Table types. */
8475         for ( ; ; ) {
8476                 struct table_type *elem;
8477
8478                 elem = TAILQ_FIRST(&p->table_types);
8479                 if (!elem)
8480                         break;
8481
8482                 TAILQ_REMOVE(&p->table_types, elem, node);
8483                 free(elem);
8484         }
8485 }
8486
8487 /*
8488  * Selector.
8489  */
8490 static struct selector *
8491 selector_find(struct rte_swx_pipeline *p, const char *name)
8492 {
8493         struct selector *s;
8494
8495         TAILQ_FOREACH(s, &p->selectors, node)
8496                 if (strcmp(s->name, name) == 0)
8497                         return s;
8498
8499         return NULL;
8500 }
8501
8502 static struct selector *
8503 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8504 {
8505         struct selector *s = NULL;
8506
8507         TAILQ_FOREACH(s, &p->selectors, node)
8508                 if (s->id == id)
8509                         return s;
8510
8511         return NULL;
8512 }
8513
8514 static int
8515 selector_fields_check(struct rte_swx_pipeline *p,
8516                       struct rte_swx_pipeline_selector_params *params,
8517                       struct header **header)
8518 {
8519         struct header *h0 = NULL;
8520         struct field *hf, *mf;
8521         uint32_t i;
8522
8523         /* Return if no selector fields. */
8524         if (!params->n_selector_fields || !params->selector_field_names)
8525                 return -EINVAL;
8526
8527         /* Check that all the selector fields either belong to the same header
8528          * or are all meta-data fields.
8529          */
8530         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8531         mf = metadata_field_parse(p, params->selector_field_names[0]);
8532         if (!hf && !mf)
8533                 return -EINVAL;
8534
8535         for (i = 1; i < params->n_selector_fields; i++)
8536                 if (h0) {
8537                         struct header *h;
8538
8539                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8540                         if (!hf || (h->id != h0->id))
8541                                 return -EINVAL;
8542                 } else {
8543                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8544                         if (!mf)
8545                                 return -EINVAL;
8546                 }
8547
8548         /* Check that there are no duplicated match fields. */
8549         for (i = 0; i < params->n_selector_fields; i++) {
8550                 const char *field_name = params->selector_field_names[i];
8551                 uint32_t j;
8552
8553                 for (j = i + 1; j < params->n_selector_fields; j++)
8554                         if (!strcmp(params->selector_field_names[j], field_name))
8555                                 return -EINVAL;
8556         }
8557
8558         /* Return. */
8559         if (header)
8560                 *header = h0;
8561
8562         return 0;
8563 }
8564
8565 int
8566 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8567                                  const char *name,
8568                                  struct rte_swx_pipeline_selector_params *params)
8569 {
8570         struct selector *s;
8571         struct header *selector_header = NULL;
8572         struct field *group_id_field, *member_id_field;
8573         uint32_t i;
8574         int status = 0;
8575
8576         CHECK(p, EINVAL);
8577
8578         CHECK_NAME(name, EINVAL);
8579         CHECK(!table_find(p, name), EEXIST);
8580         CHECK(!selector_find(p, name), EEXIST);
8581         CHECK(!learner_find(p, name), EEXIST);
8582
8583         CHECK(params, EINVAL);
8584
8585         CHECK_NAME(params->group_id_field_name, EINVAL);
8586         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8587         CHECK(group_id_field, EINVAL);
8588
8589         for (i = 0; i < params->n_selector_fields; i++) {
8590                 const char *field_name = params->selector_field_names[i];
8591
8592                 CHECK_NAME(field_name, EINVAL);
8593         }
8594         status = selector_fields_check(p, params, &selector_header);
8595         if (status)
8596                 return status;
8597
8598         CHECK_NAME(params->member_id_field_name, EINVAL);
8599         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8600         CHECK(member_id_field, EINVAL);
8601
8602         CHECK(params->n_groups_max, EINVAL);
8603
8604         CHECK(params->n_members_per_group_max, EINVAL);
8605
8606         /* Memory allocation. */
8607         s = calloc(1, sizeof(struct selector));
8608         if (!s) {
8609                 status = -ENOMEM;
8610                 goto error;
8611         }
8612
8613         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8614         if (!s->selector_fields) {
8615                 status = -ENOMEM;
8616                 goto error;
8617         }
8618
8619         /* Node initialization. */
8620         strcpy(s->name, name);
8621
8622         s->group_id_field = group_id_field;
8623
8624         for (i = 0; i < params->n_selector_fields; i++) {
8625                 const char *field_name = params->selector_field_names[i];
8626
8627                 s->selector_fields[i] = selector_header ?
8628                         header_field_parse(p, field_name, NULL) :
8629                         metadata_field_parse(p, field_name);
8630         }
8631
8632         s->n_selector_fields = params->n_selector_fields;
8633
8634         s->selector_header = selector_header;
8635
8636         s->member_id_field = member_id_field;
8637
8638         s->n_groups_max = params->n_groups_max;
8639
8640         s->n_members_per_group_max = params->n_members_per_group_max;
8641
8642         s->id = p->n_selectors;
8643
8644         /* Node add to tailq. */
8645         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8646         p->n_selectors++;
8647
8648         return 0;
8649
8650 error:
8651         if (!s)
8652                 return status;
8653
8654         free(s->selector_fields);
8655
8656         free(s);
8657
8658         return status;
8659 }
8660
8661 static void
8662 selector_params_free(struct rte_swx_table_selector_params *params)
8663 {
8664         if (!params)
8665                 return;
8666
8667         free(params->selector_mask);
8668
8669         free(params);
8670 }
8671
8672 static struct rte_swx_table_selector_params *
8673 selector_table_params_get(struct selector *s)
8674 {
8675         struct rte_swx_table_selector_params *params = NULL;
8676         struct field *first, *last;
8677         uint32_t i;
8678
8679         /* Memory allocation. */
8680         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8681         if (!params)
8682                 goto error;
8683
8684         /* Group ID. */
8685         params->group_id_offset = s->group_id_field->offset / 8;
8686
8687         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8688         first = s->selector_fields[0];
8689         last = s->selector_fields[0];
8690
8691         for (i = 0; i < s->n_selector_fields; i++) {
8692                 struct field *f = s->selector_fields[i];
8693
8694                 if (f->offset < first->offset)
8695                         first = f;
8696
8697                 if (f->offset > last->offset)
8698                         last = f;
8699         }
8700
8701         /* Selector offset and size. */
8702         params->selector_offset = first->offset / 8;
8703         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8704
8705         /* Memory allocation. */
8706         params->selector_mask = calloc(1, params->selector_size);
8707         if (!params->selector_mask)
8708                 goto error;
8709
8710         /* Selector mask. */
8711         for (i = 0; i < s->n_selector_fields; i++) {
8712                 struct field *f = s->selector_fields[i];
8713                 uint32_t start = (f->offset - first->offset) / 8;
8714                 size_t size = f->n_bits / 8;
8715
8716                 memset(&params->selector_mask[start], 0xFF, size);
8717         }
8718
8719         /* Member ID. */
8720         params->member_id_offset = s->member_id_field->offset / 8;
8721
8722         /* Maximum number of groups. */
8723         params->n_groups_max = s->n_groups_max;
8724
8725         /* Maximum number of members per group. */
8726         params->n_members_per_group_max = s->n_members_per_group_max;
8727
8728         return params;
8729
8730 error:
8731         selector_params_free(params);
8732         return NULL;
8733 }
8734
8735 static void
8736 selector_build_free(struct rte_swx_pipeline *p)
8737 {
8738         uint32_t i;
8739
8740         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8741                 struct thread *t = &p->threads[i];
8742                 uint32_t j;
8743
8744                 if (!t->selectors)
8745                         continue;
8746
8747                 for (j = 0; j < p->n_selectors; j++) {
8748                         struct selector_runtime *r = &t->selectors[j];
8749
8750                         free(r->mailbox);
8751                 }
8752
8753                 free(t->selectors);
8754                 t->selectors = NULL;
8755         }
8756
8757         free(p->selector_stats);
8758         p->selector_stats = NULL;
8759 }
8760
8761 static int
8762 selector_build(struct rte_swx_pipeline *p)
8763 {
8764         uint32_t i;
8765         int status = 0;
8766
8767         /* Per pipeline: selector statistics. */
8768         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8769         if (!p->selector_stats) {
8770                 status = -ENOMEM;
8771                 goto error;
8772         }
8773
8774         /* Per thread: selector run-time. */
8775         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8776                 struct thread *t = &p->threads[i];
8777                 struct selector *s;
8778
8779                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8780                 if (!t->selectors) {
8781                         status = -ENOMEM;
8782                         goto error;
8783                 }
8784
8785                 TAILQ_FOREACH(s, &p->selectors, node) {
8786                         struct selector_runtime *r = &t->selectors[s->id];
8787                         uint64_t size;
8788
8789                         /* r->mailbox. */
8790                         size = rte_swx_table_selector_mailbox_size_get();
8791                         if (size) {
8792                                 r->mailbox = calloc(1, size);
8793                                 if (!r->mailbox) {
8794                                         status = -ENOMEM;
8795                                         goto error;
8796                                 }
8797                         }
8798
8799                         /* r->group_id_buffer. */
8800                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8801
8802                         /* r->selector_buffer. */
8803                         r->selector_buffer = s->selector_header ?
8804                                 &t->structs[s->selector_header->struct_id] :
8805                                 &t->structs[p->metadata_struct_id];
8806
8807                         /* r->member_id_buffer. */
8808                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8809                 }
8810         }
8811
8812         return 0;
8813
8814 error:
8815         selector_build_free(p);
8816         return status;
8817 }
8818
8819 static void
8820 selector_free(struct rte_swx_pipeline *p)
8821 {
8822         selector_build_free(p);
8823
8824         /* Selector tables. */
8825         for ( ; ; ) {
8826                 struct selector *elem;
8827
8828                 elem = TAILQ_FIRST(&p->selectors);
8829                 if (!elem)
8830                         break;
8831
8832                 TAILQ_REMOVE(&p->selectors, elem, node);
8833                 free(elem->selector_fields);
8834                 free(elem);
8835         }
8836 }
8837
8838 /*
8839  * Learner table.
8840  */
8841 static struct learner *
8842 learner_find(struct rte_swx_pipeline *p, const char *name)
8843 {
8844         struct learner *l;
8845
8846         TAILQ_FOREACH(l, &p->learners, node)
8847                 if (!strcmp(l->name, name))
8848                         return l;
8849
8850         return NULL;
8851 }
8852
8853 static struct learner *
8854 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8855 {
8856         struct learner *l = NULL;
8857
8858         TAILQ_FOREACH(l, &p->learners, node)
8859                 if (l->id == id)
8860                         return l;
8861
8862         return NULL;
8863 }
8864
8865 static int
8866 learner_match_fields_check(struct rte_swx_pipeline *p,
8867                            struct rte_swx_pipeline_learner_params *params,
8868                            struct header **header)
8869 {
8870         struct header *h0 = NULL;
8871         struct field *hf, *mf;
8872         uint32_t i;
8873
8874         /* Return if no match fields. */
8875         if (!params->n_fields || !params->field_names)
8876                 return -EINVAL;
8877
8878         /* Check that all the match fields either belong to the same header
8879          * or are all meta-data fields.
8880          */
8881         hf = header_field_parse(p, params->field_names[0], &h0);
8882         mf = metadata_field_parse(p, params->field_names[0]);
8883         if (!hf && !mf)
8884                 return -EINVAL;
8885
8886         for (i = 1; i < params->n_fields; i++)
8887                 if (h0) {
8888                         struct header *h;
8889
8890                         hf = header_field_parse(p, params->field_names[i], &h);
8891                         if (!hf || (h->id != h0->id))
8892                                 return -EINVAL;
8893                 } else {
8894                         mf = metadata_field_parse(p, params->field_names[i]);
8895                         if (!mf)
8896                                 return -EINVAL;
8897                 }
8898
8899         /* Check that there are no duplicated match fields. */
8900         for (i = 0; i < params->n_fields; i++) {
8901                 const char *field_name = params->field_names[i];
8902                 uint32_t j;
8903
8904                 for (j = i + 1; j < params->n_fields; j++)
8905                         if (!strcmp(params->field_names[j], field_name))
8906                                 return -EINVAL;
8907         }
8908
8909         /* Return. */
8910         if (header)
8911                 *header = h0;
8912
8913         return 0;
8914 }
8915
8916 static int
8917 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8918 {
8919         struct struct_type *mst = p->metadata_st, *ast = a->st;
8920         struct field *mf, *af;
8921         uint32_t mf_pos, i;
8922
8923         if (!ast) {
8924                 if (mf_name)
8925                         return -EINVAL;
8926
8927                 return 0;
8928         }
8929
8930         /* Check that mf_name is the name of a valid meta-data field. */
8931         CHECK_NAME(mf_name, EINVAL);
8932         mf = metadata_field_parse(p, mf_name);
8933         CHECK(mf, EINVAL);
8934
8935         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8936          * all the action arguments.
8937          */
8938         mf_pos = mf - mst->fields;
8939         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8940
8941         /* Check that the size of each of the identified meta-data fields matches exactly the size
8942          * of the corresponding action argument.
8943          */
8944         for (i = 0; i < ast->n_fields; i++) {
8945                 mf = &mst->fields[mf_pos + i];
8946                 af = &ast->fields[i];
8947
8948                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8949         }
8950
8951         return 0;
8952 }
8953
8954 static int
8955 learner_action_learning_check(struct rte_swx_pipeline *p,
8956                               struct action *action,
8957                               const char **action_names,
8958                               uint32_t n_actions)
8959 {
8960         uint32_t i;
8961
8962         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8963          * the action passed as argument to the "learn" instruction) is also enabled for the
8964          * current learner table.
8965          */
8966         for (i = 0; i < action->n_instructions; i++) {
8967                 struct instruction *instr = &action->instructions[i];
8968                 uint32_t found = 0, j;
8969
8970                 if (instr->type != INSTR_LEARNER_LEARN)
8971                         continue;
8972
8973                 for (j = 0; j < n_actions; j++) {
8974                         struct action *a;
8975
8976                         a = action_find(p, action_names[j]);
8977                         if (!a)
8978                                 return -EINVAL;
8979
8980                         if (a->id == instr->learn.action_id)
8981                                 found = 1;
8982                 }
8983
8984                 if (!found)
8985                         return -EINVAL;
8986         }
8987
8988         return 0;
8989 }
8990
8991 int
8992 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8993                               const char *name,
8994                               struct rte_swx_pipeline_learner_params *params,
8995                               uint32_t size,
8996                               uint32_t timeout)
8997 {
8998         struct learner *l = NULL;
8999         struct action *default_action;
9000         struct header *header = NULL;
9001         uint32_t action_data_size_max = 0, i;
9002         int status = 0;
9003
9004         CHECK(p, EINVAL);
9005
9006         CHECK_NAME(name, EINVAL);
9007         CHECK(!table_find(p, name), EEXIST);
9008         CHECK(!selector_find(p, name), EEXIST);
9009         CHECK(!learner_find(p, name), EEXIST);
9010
9011         CHECK(params, EINVAL);
9012
9013         /* Match checks. */
9014         status = learner_match_fields_check(p, params, &header);
9015         if (status)
9016                 return status;
9017
9018         /* Action checks. */
9019         CHECK(params->n_actions, EINVAL);
9020
9021         CHECK(params->action_names, EINVAL);
9022         for (i = 0; i < params->n_actions; i++) {
9023                 const char *action_name = params->action_names[i];
9024                 const char *action_field_name = params->action_field_names[i];
9025                 struct action *a;
9026                 uint32_t action_data_size;
9027
9028                 CHECK_NAME(action_name, EINVAL);
9029
9030                 a = action_find(p, action_name);
9031                 CHECK(a, EINVAL);
9032
9033                 status = learner_action_args_check(p, a, action_field_name);
9034                 if (status)
9035                         return status;
9036
9037                 status = learner_action_learning_check(p,
9038                                                        a,
9039                                                        params->action_names,
9040                                                        params->n_actions);
9041                 if (status)
9042                         return status;
9043
9044                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9045                 if (action_data_size > action_data_size_max)
9046                         action_data_size_max = action_data_size;
9047         }
9048
9049         CHECK_NAME(params->default_action_name, EINVAL);
9050         for (i = 0; i < p->n_actions; i++)
9051                 if (!strcmp(params->action_names[i],
9052                             params->default_action_name))
9053                         break;
9054         CHECK(i < params->n_actions, EINVAL);
9055
9056         default_action = action_find(p, params->default_action_name);
9057         CHECK((default_action->st && params->default_action_data) ||
9058               !params->default_action_data, EINVAL);
9059
9060         /* Any other checks. */
9061         CHECK(size, EINVAL);
9062         CHECK(timeout, EINVAL);
9063
9064         /* Memory allocation. */
9065         l = calloc(1, sizeof(struct learner));
9066         if (!l)
9067                 goto nomem;
9068
9069         l->fields = calloc(params->n_fields, sizeof(struct field *));
9070         if (!l->fields)
9071                 goto nomem;
9072
9073         l->actions = calloc(params->n_actions, sizeof(struct action *));
9074         if (!l->actions)
9075                 goto nomem;
9076
9077         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
9078         if (!l->action_arg)
9079                 goto nomem;
9080
9081         if (action_data_size_max) {
9082                 l->default_action_data = calloc(1, action_data_size_max);
9083                 if (!l->default_action_data)
9084                         goto nomem;
9085         }
9086
9087         /* Node initialization. */
9088         strcpy(l->name, name);
9089
9090         for (i = 0; i < params->n_fields; i++) {
9091                 const char *field_name = params->field_names[i];
9092
9093                 l->fields[i] = header ?
9094                         header_field_parse(p, field_name, NULL) :
9095                         metadata_field_parse(p, field_name);
9096         }
9097
9098         l->n_fields = params->n_fields;
9099
9100         l->header = header;
9101
9102         for (i = 0; i < params->n_actions; i++) {
9103                 const char *mf_name = params->action_field_names[i];
9104
9105                 l->actions[i] = action_find(p, params->action_names[i]);
9106
9107                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
9108         }
9109
9110         l->default_action = default_action;
9111
9112         if (default_action->st)
9113                 memcpy(l->default_action_data,
9114                        params->default_action_data,
9115                        default_action->st->n_bits / 8);
9116
9117         l->n_actions = params->n_actions;
9118
9119         l->default_action_is_const = params->default_action_is_const;
9120
9121         l->action_data_size_max = action_data_size_max;
9122
9123         l->size = size;
9124
9125         l->timeout = timeout;
9126
9127         l->id = p->n_learners;
9128
9129         /* Node add to tailq. */
9130         TAILQ_INSERT_TAIL(&p->learners, l, node);
9131         p->n_learners++;
9132
9133         return 0;
9134
9135 nomem:
9136         if (!l)
9137                 return -ENOMEM;
9138
9139         free(l->action_arg);
9140         free(l->actions);
9141         free(l->fields);
9142         free(l);
9143
9144         return -ENOMEM;
9145 }
9146
9147 static void
9148 learner_params_free(struct rte_swx_table_learner_params *params)
9149 {
9150         if (!params)
9151                 return;
9152
9153         free(params->key_mask0);
9154
9155         free(params);
9156 }
9157
9158 static struct rte_swx_table_learner_params *
9159 learner_params_get(struct learner *l)
9160 {
9161         struct rte_swx_table_learner_params *params = NULL;
9162         struct field *first, *last;
9163         uint32_t i;
9164
9165         /* Memory allocation. */
9166         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
9167         if (!params)
9168                 goto error;
9169
9170         /* Find first (smallest offset) and last (biggest offset) match fields. */
9171         first = l->fields[0];
9172         last = l->fields[0];
9173
9174         for (i = 0; i < l->n_fields; i++) {
9175                 struct field *f = l->fields[i];
9176
9177                 if (f->offset < first->offset)
9178                         first = f;
9179
9180                 if (f->offset > last->offset)
9181                         last = f;
9182         }
9183
9184         /* Key offset and size. */
9185         params->key_offset = first->offset / 8;
9186         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9187
9188         /* Memory allocation. */
9189         params->key_mask0 = calloc(1, params->key_size);
9190         if (!params->key_mask0)
9191                 goto error;
9192
9193         /* Key mask. */
9194         for (i = 0; i < l->n_fields; i++) {
9195                 struct field *f = l->fields[i];
9196                 uint32_t start = (f->offset - first->offset) / 8;
9197                 size_t size = f->n_bits / 8;
9198
9199                 memset(&params->key_mask0[start], 0xFF, size);
9200         }
9201
9202         /* Action data size. */
9203         params->action_data_size = l->action_data_size_max;
9204
9205         /* Maximum number of keys. */
9206         params->n_keys_max = l->size;
9207
9208         /* Timeout. */
9209         params->key_timeout = l->timeout;
9210
9211         return params;
9212
9213 error:
9214         learner_params_free(params);
9215         return NULL;
9216 }
9217
9218 static void
9219 learner_build_free(struct rte_swx_pipeline *p)
9220 {
9221         uint32_t i;
9222
9223         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9224                 struct thread *t = &p->threads[i];
9225                 uint32_t j;
9226
9227                 if (!t->learners)
9228                         continue;
9229
9230                 for (j = 0; j < p->n_learners; j++) {
9231                         struct learner_runtime *r = &t->learners[j];
9232
9233                         free(r->mailbox);
9234                         free(r->action_data);
9235                 }
9236
9237                 free(t->learners);
9238                 t->learners = NULL;
9239         }
9240
9241         if (p->learner_stats) {
9242                 for (i = 0; i < p->n_learners; i++)
9243                         free(p->learner_stats[i].n_pkts_action);
9244
9245                 free(p->learner_stats);
9246         }
9247 }
9248
9249 static int
9250 learner_build(struct rte_swx_pipeline *p)
9251 {
9252         uint32_t i;
9253         int status = 0;
9254
9255         /* Per pipeline: learner statistics. */
9256         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9257         CHECK(p->learner_stats, ENOMEM);
9258
9259         for (i = 0; i < p->n_learners; i++) {
9260                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9261                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9262         }
9263
9264         /* Per thread: learner run-time. */
9265         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9266                 struct thread *t = &p->threads[i];
9267                 struct learner *l;
9268
9269                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9270                 if (!t->learners) {
9271                         status = -ENOMEM;
9272                         goto error;
9273                 }
9274
9275                 TAILQ_FOREACH(l, &p->learners, node) {
9276                         struct learner_runtime *r = &t->learners[l->id];
9277                         uint64_t size;
9278                         uint32_t j;
9279
9280                         /* r->mailbox. */
9281                         size = rte_swx_table_learner_mailbox_size_get();
9282                         if (size) {
9283                                 r->mailbox = calloc(1, size);
9284                                 if (!r->mailbox) {
9285                                         status = -ENOMEM;
9286                                         goto error;
9287                                 }
9288                         }
9289
9290                         /* r->key. */
9291                         r->key = l->header ?
9292                                 &t->structs[l->header->struct_id] :
9293                                 &t->structs[p->metadata_struct_id];
9294
9295                         /* r->action_data. */
9296                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
9297                         if (!r->action_data) {
9298                                 status = -ENOMEM;
9299                                 goto error;
9300                         }
9301
9302                         for (j = 0; j < l->n_actions; j++) {
9303                                 struct action *a = l->actions[j];
9304                                 struct field *mf = l->action_arg[j];
9305                                 uint8_t *m = t->structs[p->metadata_struct_id];
9306
9307                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
9308                         }
9309                 }
9310         }
9311
9312         return 0;
9313
9314 error:
9315         learner_build_free(p);
9316         return status;
9317 }
9318
9319 static void
9320 learner_free(struct rte_swx_pipeline *p)
9321 {
9322         learner_build_free(p);
9323
9324         /* Learner tables. */
9325         for ( ; ; ) {
9326                 struct learner *l;
9327
9328                 l = TAILQ_FIRST(&p->learners);
9329                 if (!l)
9330                         break;
9331
9332                 TAILQ_REMOVE(&p->learners, l, node);
9333                 free(l->fields);
9334                 free(l->actions);
9335                 free(l->action_arg);
9336                 free(l->default_action_data);
9337                 free(l);
9338         }
9339 }
9340
9341 /*
9342  * Table state.
9343  */
9344 static int
9345 table_state_build(struct rte_swx_pipeline *p)
9346 {
9347         struct table *table;
9348         struct selector *s;
9349         struct learner *l;
9350
9351         p->table_state = calloc(p->n_tables + p->n_selectors,
9352                                 sizeof(struct rte_swx_table_state));
9353         CHECK(p->table_state, ENOMEM);
9354
9355         TAILQ_FOREACH(table, &p->tables, node) {
9356                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9357
9358                 if (table->type) {
9359                         struct rte_swx_table_params *params;
9360
9361                         /* ts->obj. */
9362                         params = table_params_get(table);
9363                         CHECK(params, ENOMEM);
9364
9365                         ts->obj = table->type->ops.create(params,
9366                                 NULL,
9367                                 table->args,
9368                                 p->numa_node);
9369
9370                         table_params_free(params);
9371                         CHECK(ts->obj, ENODEV);
9372                 }
9373
9374                 /* ts->default_action_data. */
9375                 if (table->action_data_size_max) {
9376                         ts->default_action_data =
9377                                 malloc(table->action_data_size_max);
9378                         CHECK(ts->default_action_data, ENOMEM);
9379
9380                         memcpy(ts->default_action_data,
9381                                table->default_action_data,
9382                                table->action_data_size_max);
9383                 }
9384
9385                 /* ts->default_action_id. */
9386                 ts->default_action_id = table->default_action->id;
9387         }
9388
9389         TAILQ_FOREACH(s, &p->selectors, node) {
9390                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9391                 struct rte_swx_table_selector_params *params;
9392
9393                 /* ts->obj. */
9394                 params = selector_table_params_get(s);
9395                 CHECK(params, ENOMEM);
9396
9397                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9398
9399                 selector_params_free(params);
9400                 CHECK(ts->obj, ENODEV);
9401         }
9402
9403         TAILQ_FOREACH(l, &p->learners, node) {
9404                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9405                         p->n_selectors + l->id];
9406                 struct rte_swx_table_learner_params *params;
9407
9408                 /* ts->obj. */
9409                 params = learner_params_get(l);
9410                 CHECK(params, ENOMEM);
9411
9412                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9413                 learner_params_free(params);
9414                 CHECK(ts->obj, ENODEV);
9415
9416                 /* ts->default_action_data. */
9417                 if (l->action_data_size_max) {
9418                         ts->default_action_data = malloc(l->action_data_size_max);
9419                         CHECK(ts->default_action_data, ENOMEM);
9420
9421                         memcpy(ts->default_action_data,
9422                                l->default_action_data,
9423                                l->action_data_size_max);
9424                 }
9425
9426                 /* ts->default_action_id. */
9427                 ts->default_action_id = l->default_action->id;
9428         }
9429
9430         return 0;
9431 }
9432
9433 static void
9434 table_state_build_free(struct rte_swx_pipeline *p)
9435 {
9436         uint32_t i;
9437
9438         if (!p->table_state)
9439                 return;
9440
9441         for (i = 0; i < p->n_tables; i++) {
9442                 struct rte_swx_table_state *ts = &p->table_state[i];
9443                 struct table *table = table_find_by_id(p, i);
9444
9445                 /* ts->obj. */
9446                 if (table->type && ts->obj)
9447                         table->type->ops.free(ts->obj);
9448
9449                 /* ts->default_action_data. */
9450                 free(ts->default_action_data);
9451         }
9452
9453         for (i = 0; i < p->n_selectors; i++) {
9454                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9455
9456                 /* ts->obj. */
9457                 if (ts->obj)
9458                         rte_swx_table_selector_free(ts->obj);
9459         }
9460
9461         for (i = 0; i < p->n_learners; i++) {
9462                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9463
9464                 /* ts->obj. */
9465                 if (ts->obj)
9466                         rte_swx_table_learner_free(ts->obj);
9467
9468                 /* ts->default_action_data. */
9469                 free(ts->default_action_data);
9470         }
9471
9472         free(p->table_state);
9473         p->table_state = NULL;
9474 }
9475
9476 static void
9477 table_state_free(struct rte_swx_pipeline *p)
9478 {
9479         table_state_build_free(p);
9480 }
9481
9482 /*
9483  * Register array.
9484  */
9485 static struct regarray *
9486 regarray_find(struct rte_swx_pipeline *p, const char *name)
9487 {
9488         struct regarray *elem;
9489
9490         TAILQ_FOREACH(elem, &p->regarrays, node)
9491                 if (!strcmp(elem->name, name))
9492                         return elem;
9493
9494         return NULL;
9495 }
9496
9497 static struct regarray *
9498 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9499 {
9500         struct regarray *elem = NULL;
9501
9502         TAILQ_FOREACH(elem, &p->regarrays, node)
9503                 if (elem->id == id)
9504                         return elem;
9505
9506         return NULL;
9507 }
9508
9509 int
9510 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9511                               const char *name,
9512                               uint32_t size,
9513                               uint64_t init_val)
9514 {
9515         struct regarray *r;
9516
9517         CHECK(p, EINVAL);
9518
9519         CHECK_NAME(name, EINVAL);
9520         CHECK(!regarray_find(p, name), EEXIST);
9521
9522         CHECK(size, EINVAL);
9523         size = rte_align32pow2(size);
9524
9525         /* Memory allocation. */
9526         r = calloc(1, sizeof(struct regarray));
9527         CHECK(r, ENOMEM);
9528
9529         /* Node initialization. */
9530         strcpy(r->name, name);
9531         r->init_val = init_val;
9532         r->size = size;
9533         r->id = p->n_regarrays;
9534
9535         /* Node add to tailq. */
9536         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9537         p->n_regarrays++;
9538
9539         return 0;
9540 }
9541
9542 static int
9543 regarray_build(struct rte_swx_pipeline *p)
9544 {
9545         struct regarray *regarray;
9546
9547         if (!p->n_regarrays)
9548                 return 0;
9549
9550         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9551         CHECK(p->regarray_runtime, ENOMEM);
9552
9553         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9554                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9555                 uint32_t i;
9556
9557                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9558                                          RTE_CACHE_LINE_SIZE,
9559                                          p->numa_node);
9560                 CHECK(r->regarray, ENOMEM);
9561
9562                 if (regarray->init_val)
9563                         for (i = 0; i < regarray->size; i++)
9564                                 r->regarray[i] = regarray->init_val;
9565
9566                 r->size_mask = regarray->size - 1;
9567         }
9568
9569         return 0;
9570 }
9571
9572 static void
9573 regarray_build_free(struct rte_swx_pipeline *p)
9574 {
9575         uint32_t i;
9576
9577         if (!p->regarray_runtime)
9578                 return;
9579
9580         for (i = 0; i < p->n_regarrays; i++) {
9581                 struct regarray *regarray = regarray_find_by_id(p, i);
9582                 struct regarray_runtime *r = &p->regarray_runtime[i];
9583
9584                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9585         }
9586
9587         free(p->regarray_runtime);
9588         p->regarray_runtime = NULL;
9589 }
9590
9591 static void
9592 regarray_free(struct rte_swx_pipeline *p)
9593 {
9594         regarray_build_free(p);
9595
9596         for ( ; ; ) {
9597                 struct regarray *elem;
9598
9599                 elem = TAILQ_FIRST(&p->regarrays);
9600                 if (!elem)
9601                         break;
9602
9603                 TAILQ_REMOVE(&p->regarrays, elem, node);
9604                 free(elem);
9605         }
9606 }
9607
9608 /*
9609  * Meter array.
9610  */
9611 static struct meter_profile *
9612 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9613 {
9614         struct meter_profile *elem;
9615
9616         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9617                 if (!strcmp(elem->name, name))
9618                         return elem;
9619
9620         return NULL;
9621 }
9622
9623 static struct metarray *
9624 metarray_find(struct rte_swx_pipeline *p, const char *name)
9625 {
9626         struct metarray *elem;
9627
9628         TAILQ_FOREACH(elem, &p->metarrays, node)
9629                 if (!strcmp(elem->name, name))
9630                         return elem;
9631
9632         return NULL;
9633 }
9634
9635 static struct metarray *
9636 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9637 {
9638         struct metarray *elem = NULL;
9639
9640         TAILQ_FOREACH(elem, &p->metarrays, node)
9641                 if (elem->id == id)
9642                         return elem;
9643
9644         return NULL;
9645 }
9646
9647 int
9648 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9649                                  const char *name,
9650                                  uint32_t size)
9651 {
9652         struct metarray *m;
9653
9654         CHECK(p, EINVAL);
9655
9656         CHECK_NAME(name, EINVAL);
9657         CHECK(!metarray_find(p, name), EEXIST);
9658
9659         CHECK(size, EINVAL);
9660         size = rte_align32pow2(size);
9661
9662         /* Memory allocation. */
9663         m = calloc(1, sizeof(struct metarray));
9664         CHECK(m, ENOMEM);
9665
9666         /* Node initialization. */
9667         strcpy(m->name, name);
9668         m->size = size;
9669         m->id = p->n_metarrays;
9670
9671         /* Node add to tailq. */
9672         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9673         p->n_metarrays++;
9674
9675         return 0;
9676 }
9677
9678 struct meter_profile meter_profile_default = {
9679         .node = {0},
9680         .name = "",
9681         .params = {0},
9682
9683         .profile = {
9684                 .cbs = 10000,
9685                 .pbs = 10000,
9686                 .cir_period = 1,
9687                 .cir_bytes_per_period = 1,
9688                 .pir_period = 1,
9689                 .pir_bytes_per_period = 1,
9690         },
9691
9692         .n_users = 0,
9693 };
9694
9695 static void
9696 meter_init(struct meter *m)
9697 {
9698         memset(m, 0, sizeof(struct meter));
9699         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9700         m->profile = &meter_profile_default;
9701         m->color_mask = RTE_COLOR_GREEN;
9702
9703         meter_profile_default.n_users++;
9704 }
9705
9706 static int
9707 metarray_build(struct rte_swx_pipeline *p)
9708 {
9709         struct metarray *m;
9710
9711         if (!p->n_metarrays)
9712                 return 0;
9713
9714         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9715         CHECK(p->metarray_runtime, ENOMEM);
9716
9717         TAILQ_FOREACH(m, &p->metarrays, node) {
9718                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9719                 uint32_t i;
9720
9721                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9722                                          RTE_CACHE_LINE_SIZE,
9723                                          p->numa_node);
9724                 CHECK(r->metarray, ENOMEM);
9725
9726                 for (i = 0; i < m->size; i++)
9727                         meter_init(&r->metarray[i]);
9728
9729                 r->size_mask = m->size - 1;
9730         }
9731
9732         return 0;
9733 }
9734
9735 static void
9736 metarray_build_free(struct rte_swx_pipeline *p)
9737 {
9738         uint32_t i;
9739
9740         if (!p->metarray_runtime)
9741                 return;
9742
9743         for (i = 0; i < p->n_metarrays; i++) {
9744                 struct metarray *m = metarray_find_by_id(p, i);
9745                 struct metarray_runtime *r = &p->metarray_runtime[i];
9746
9747                 env_free(r->metarray, m->size * sizeof(struct meter));
9748         }
9749
9750         free(p->metarray_runtime);
9751         p->metarray_runtime = NULL;
9752 }
9753
9754 static void
9755 metarray_free(struct rte_swx_pipeline *p)
9756 {
9757         metarray_build_free(p);
9758
9759         /* Meter arrays. */
9760         for ( ; ; ) {
9761                 struct metarray *elem;
9762
9763                 elem = TAILQ_FIRST(&p->metarrays);
9764                 if (!elem)
9765                         break;
9766
9767                 TAILQ_REMOVE(&p->metarrays, elem, node);
9768                 free(elem);
9769         }
9770
9771         /* Meter profiles. */
9772         for ( ; ; ) {
9773                 struct meter_profile *elem;
9774
9775                 elem = TAILQ_FIRST(&p->meter_profiles);
9776                 if (!elem)
9777                         break;
9778
9779                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9780                 free(elem);
9781         }
9782 }
9783
9784 /*
9785  * Pipeline.
9786  */
9787 int
9788 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9789 {
9790         struct rte_swx_pipeline *pipeline;
9791
9792         /* Check input parameters. */
9793         CHECK(p, EINVAL);
9794
9795         /* Memory allocation. */
9796         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9797         CHECK(pipeline, ENOMEM);
9798
9799         /* Initialization. */
9800         TAILQ_INIT(&pipeline->struct_types);
9801         TAILQ_INIT(&pipeline->port_in_types);
9802         TAILQ_INIT(&pipeline->ports_in);
9803         TAILQ_INIT(&pipeline->port_out_types);
9804         TAILQ_INIT(&pipeline->ports_out);
9805         TAILQ_INIT(&pipeline->extern_types);
9806         TAILQ_INIT(&pipeline->extern_objs);
9807         TAILQ_INIT(&pipeline->extern_funcs);
9808         TAILQ_INIT(&pipeline->headers);
9809         TAILQ_INIT(&pipeline->actions);
9810         TAILQ_INIT(&pipeline->table_types);
9811         TAILQ_INIT(&pipeline->tables);
9812         TAILQ_INIT(&pipeline->selectors);
9813         TAILQ_INIT(&pipeline->learners);
9814         TAILQ_INIT(&pipeline->regarrays);
9815         TAILQ_INIT(&pipeline->meter_profiles);
9816         TAILQ_INIT(&pipeline->metarrays);
9817
9818         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9819         pipeline->numa_node = numa_node;
9820
9821         *p = pipeline;
9822         return 0;
9823 }
9824
9825 void
9826 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9827 {
9828         if (!p)
9829                 return;
9830
9831         free(p->instructions);
9832
9833         metarray_free(p);
9834         regarray_free(p);
9835         table_state_free(p);
9836         learner_free(p);
9837         selector_free(p);
9838         table_free(p);
9839         action_free(p);
9840         metadata_free(p);
9841         header_free(p);
9842         extern_func_free(p);
9843         extern_obj_free(p);
9844         port_out_free(p);
9845         port_in_free(p);
9846         struct_free(p);
9847
9848         free(p);
9849 }
9850
9851 int
9852 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9853                                      const char **instructions,
9854                                      uint32_t n_instructions)
9855 {
9856         int err;
9857         uint32_t i;
9858
9859         err = instruction_config(p, NULL, instructions, n_instructions);
9860         if (err)
9861                 return err;
9862
9863         /* Thread instruction pointer reset. */
9864         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9865                 struct thread *t = &p->threads[i];
9866
9867                 thread_ip_reset(p, t);
9868         }
9869
9870         return 0;
9871 }
9872
9873 int
9874 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9875 {
9876         int status;
9877
9878         CHECK(p, EINVAL);
9879         CHECK(p->build_done == 0, EEXIST);
9880
9881         status = port_in_build(p);
9882         if (status)
9883                 goto error;
9884
9885         status = port_out_build(p);
9886         if (status)
9887                 goto error;
9888
9889         status = struct_build(p);
9890         if (status)
9891                 goto error;
9892
9893         status = extern_obj_build(p);
9894         if (status)
9895                 goto error;
9896
9897         status = extern_func_build(p);
9898         if (status)
9899                 goto error;
9900
9901         status = header_build(p);
9902         if (status)
9903                 goto error;
9904
9905         status = metadata_build(p);
9906         if (status)
9907                 goto error;
9908
9909         status = action_build(p);
9910         if (status)
9911                 goto error;
9912
9913         status = table_build(p);
9914         if (status)
9915                 goto error;
9916
9917         status = selector_build(p);
9918         if (status)
9919                 goto error;
9920
9921         status = learner_build(p);
9922         if (status)
9923                 goto error;
9924
9925         status = table_state_build(p);
9926         if (status)
9927                 goto error;
9928
9929         status = regarray_build(p);
9930         if (status)
9931                 goto error;
9932
9933         status = metarray_build(p);
9934         if (status)
9935                 goto error;
9936
9937         p->build_done = 1;
9938         return 0;
9939
9940 error:
9941         metarray_build_free(p);
9942         regarray_build_free(p);
9943         table_state_build_free(p);
9944         learner_build_free(p);
9945         selector_build_free(p);
9946         table_build_free(p);
9947         action_build_free(p);
9948         metadata_build_free(p);
9949         header_build_free(p);
9950         extern_func_build_free(p);
9951         extern_obj_build_free(p);
9952         port_out_build_free(p);
9953         port_in_build_free(p);
9954         struct_build_free(p);
9955
9956         return status;
9957 }
9958
9959 void
9960 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9961 {
9962         uint32_t i;
9963
9964         for (i = 0; i < n_instructions; i++)
9965                 instr_exec(p);
9966 }
9967
9968 void
9969 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9970 {
9971         uint32_t i;
9972
9973         for (i = 0; i < p->n_ports_out; i++) {
9974                 struct port_out_runtime *port = &p->out[i];
9975
9976                 if (port->flush)
9977                         port->flush(port->obj);
9978         }
9979 }
9980
9981 /*
9982  * Control.
9983  */
9984 int
9985 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9986                               struct rte_swx_ctl_pipeline_info *pipeline)
9987 {
9988         struct action *action;
9989         struct table *table;
9990         uint32_t n_actions = 0, n_tables = 0;
9991
9992         if (!p || !pipeline)
9993                 return -EINVAL;
9994
9995         TAILQ_FOREACH(action, &p->actions, node)
9996                 n_actions++;
9997
9998         TAILQ_FOREACH(table, &p->tables, node)
9999                 n_tables++;
10000
10001         pipeline->n_ports_in = p->n_ports_in;
10002         pipeline->n_ports_out = p->n_ports_out;
10003         pipeline->n_actions = n_actions;
10004         pipeline->n_tables = n_tables;
10005         pipeline->n_selectors = p->n_selectors;
10006         pipeline->n_learners = p->n_learners;
10007         pipeline->n_regarrays = p->n_regarrays;
10008         pipeline->n_metarrays = p->n_metarrays;
10009
10010         return 0;
10011 }
10012
10013 int
10014 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10015 {
10016         if (!p || !numa_node)
10017                 return -EINVAL;
10018
10019         *numa_node = p->numa_node;
10020         return 0;
10021 }
10022
10023 int
10024 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10025                             uint32_t action_id,
10026                             struct rte_swx_ctl_action_info *action)
10027 {
10028         struct action *a = NULL;
10029
10030         if (!p || (action_id >= p->n_actions) || !action)
10031                 return -EINVAL;
10032
10033         a = action_find_by_id(p, action_id);
10034         if (!a)
10035                 return -EINVAL;
10036
10037         strcpy(action->name, a->name);
10038         action->n_args = a->st ? a->st->n_fields : 0;
10039         return 0;
10040 }
10041
10042 int
10043 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10044                                 uint32_t action_id,
10045                                 uint32_t action_arg_id,
10046                                 struct rte_swx_ctl_action_arg_info *action_arg)
10047 {
10048         struct action *a = NULL;
10049         struct field *arg = NULL;
10050
10051         if (!p || (action_id >= p->n_actions) || !action_arg)
10052                 return -EINVAL;
10053
10054         a = action_find_by_id(p, action_id);
10055         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10056                 return -EINVAL;
10057
10058         arg = &a->st->fields[action_arg_id];
10059         strcpy(action_arg->name, arg->name);
10060         action_arg->n_bits = arg->n_bits;
10061         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10062
10063         return 0;
10064 }
10065
10066 int
10067 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10068                            uint32_t table_id,
10069                            struct rte_swx_ctl_table_info *table)
10070 {
10071         struct table *t = NULL;
10072
10073         if (!p || !table)
10074                 return -EINVAL;
10075
10076         t = table_find_by_id(p, table_id);
10077         if (!t)
10078                 return -EINVAL;
10079
10080         strcpy(table->name, t->name);
10081         strcpy(table->args, t->args);
10082         table->n_match_fields = t->n_fields;
10083         table->n_actions = t->n_actions;
10084         table->default_action_is_const = t->default_action_is_const;
10085         table->size = t->size;
10086         return 0;
10087 }
10088
10089 int
10090 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10091         uint32_t table_id,
10092         uint32_t match_field_id,
10093         struct rte_swx_ctl_table_match_field_info *match_field)
10094 {
10095         struct table *t;
10096         struct match_field *f;
10097
10098         if (!p || (table_id >= p->n_tables) || !match_field)
10099                 return -EINVAL;
10100
10101         t = table_find_by_id(p, table_id);
10102         if (!t || (match_field_id >= t->n_fields))
10103                 return -EINVAL;
10104
10105         f = &t->fields[match_field_id];
10106         match_field->match_type = f->match_type;
10107         match_field->is_header = t->header ? 1 : 0;
10108         match_field->n_bits = f->field->n_bits;
10109         match_field->offset = f->field->offset;
10110
10111         return 0;
10112 }
10113
10114 int
10115 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10116         uint32_t table_id,
10117         uint32_t table_action_id,
10118         struct rte_swx_ctl_table_action_info *table_action)
10119 {
10120         struct table *t;
10121
10122         if (!p || (table_id >= p->n_tables) || !table_action)
10123                 return -EINVAL;
10124
10125         t = table_find_by_id(p, table_id);
10126         if (!t || (table_action_id >= t->n_actions))
10127                 return -EINVAL;
10128
10129         table_action->action_id = t->actions[table_action_id]->id;
10130
10131         return 0;
10132 }
10133
10134 int
10135 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10136                           uint32_t table_id,
10137                           struct rte_swx_table_ops *table_ops,
10138                           int *is_stub)
10139 {
10140         struct table *t;
10141
10142         if (!p || (table_id >= p->n_tables))
10143                 return -EINVAL;
10144
10145         t = table_find_by_id(p, table_id);
10146         if (!t)
10147                 return -EINVAL;
10148
10149         if (t->type) {
10150                 if (table_ops)
10151                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10152                 *is_stub = 0;
10153         } else {
10154                 *is_stub = 1;
10155         }
10156
10157         return 0;
10158 }
10159
10160 int
10161 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10162                               uint32_t selector_id,
10163                               struct rte_swx_ctl_selector_info *selector)
10164 {
10165         struct selector *s = NULL;
10166
10167         if (!p || !selector)
10168                 return -EINVAL;
10169
10170         s = selector_find_by_id(p, selector_id);
10171         if (!s)
10172                 return -EINVAL;
10173
10174         strcpy(selector->name, s->name);
10175
10176         selector->n_selector_fields = s->n_selector_fields;
10177         selector->n_groups_max = s->n_groups_max;
10178         selector->n_members_per_group_max = s->n_members_per_group_max;
10179
10180         return 0;
10181 }
10182
10183 int
10184 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10185          uint32_t selector_id,
10186          struct rte_swx_ctl_table_match_field_info *field)
10187 {
10188         struct selector *s;
10189
10190         if (!p || (selector_id >= p->n_selectors) || !field)
10191                 return -EINVAL;
10192
10193         s = selector_find_by_id(p, selector_id);
10194         if (!s)
10195                 return -EINVAL;
10196
10197         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10198         field->is_header = 0;
10199         field->n_bits = s->group_id_field->n_bits;
10200         field->offset = s->group_id_field->offset;
10201
10202         return 0;
10203 }
10204
10205 int
10206 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10207          uint32_t selector_id,
10208          uint32_t selector_field_id,
10209          struct rte_swx_ctl_table_match_field_info *field)
10210 {
10211         struct selector *s;
10212         struct field *f;
10213
10214         if (!p || (selector_id >= p->n_selectors) || !field)
10215                 return -EINVAL;
10216
10217         s = selector_find_by_id(p, selector_id);
10218         if (!s || (selector_field_id >= s->n_selector_fields))
10219                 return -EINVAL;
10220
10221         f = s->selector_fields[selector_field_id];
10222         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10223         field->is_header = s->selector_header ? 1 : 0;
10224         field->n_bits = f->n_bits;
10225         field->offset = f->offset;
10226
10227         return 0;
10228 }
10229
10230 int
10231 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10232          uint32_t selector_id,
10233          struct rte_swx_ctl_table_match_field_info *field)
10234 {
10235         struct selector *s;
10236
10237         if (!p || (selector_id >= p->n_selectors) || !field)
10238                 return -EINVAL;
10239
10240         s = selector_find_by_id(p, selector_id);
10241         if (!s)
10242                 return -EINVAL;
10243
10244         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10245         field->is_header = 0;
10246         field->n_bits = s->member_id_field->n_bits;
10247         field->offset = s->member_id_field->offset;
10248
10249         return 0;
10250 }
10251
10252 int
10253 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10254                              uint32_t learner_id,
10255                              struct rte_swx_ctl_learner_info *learner)
10256 {
10257         struct learner *l = NULL;
10258
10259         if (!p || !learner)
10260                 return -EINVAL;
10261
10262         l = learner_find_by_id(p, learner_id);
10263         if (!l)
10264                 return -EINVAL;
10265
10266         strcpy(learner->name, l->name);
10267
10268         learner->n_match_fields = l->n_fields;
10269         learner->n_actions = l->n_actions;
10270         learner->default_action_is_const = l->default_action_is_const;
10271         learner->size = l->size;
10272
10273         return 0;
10274 }
10275
10276 int
10277 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10278                                          uint32_t learner_id,
10279                                          uint32_t match_field_id,
10280                                          struct rte_swx_ctl_table_match_field_info *match_field)
10281 {
10282         struct learner *l;
10283         struct field *f;
10284
10285         if (!p || (learner_id >= p->n_learners) || !match_field)
10286                 return -EINVAL;
10287
10288         l = learner_find_by_id(p, learner_id);
10289         if (!l || (match_field_id >= l->n_fields))
10290                 return -EINVAL;
10291
10292         f = l->fields[match_field_id];
10293         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10294         match_field->is_header = l->header ? 1 : 0;
10295         match_field->n_bits = f->n_bits;
10296         match_field->offset = f->offset;
10297
10298         return 0;
10299 }
10300
10301 int
10302 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10303                                     uint32_t learner_id,
10304                                     uint32_t learner_action_id,
10305                                     struct rte_swx_ctl_table_action_info *learner_action)
10306 {
10307         struct learner *l;
10308
10309         if (!p || (learner_id >= p->n_learners) || !learner_action)
10310                 return -EINVAL;
10311
10312         l = learner_find_by_id(p, learner_id);
10313         if (!l || (learner_action_id >= l->n_actions))
10314                 return -EINVAL;
10315
10316         learner_action->action_id = l->actions[learner_action_id]->id;
10317
10318         return 0;
10319 }
10320
10321 int
10322 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10323                                  struct rte_swx_table_state **table_state)
10324 {
10325         if (!p || !table_state || !p->build_done)
10326                 return -EINVAL;
10327
10328         *table_state = p->table_state;
10329         return 0;
10330 }
10331
10332 int
10333 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10334                                  struct rte_swx_table_state *table_state)
10335 {
10336         if (!p || !table_state || !p->build_done)
10337                 return -EINVAL;
10338
10339         p->table_state = table_state;
10340         return 0;
10341 }
10342
10343 int
10344 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10345                                         uint32_t port_id,
10346                                         struct rte_swx_port_in_stats *stats)
10347 {
10348         struct port_in *port;
10349
10350         if (!p || !stats)
10351                 return -EINVAL;
10352
10353         port = port_in_find(p, port_id);
10354         if (!port)
10355                 return -EINVAL;
10356
10357         port->type->ops.stats_read(port->obj, stats);
10358         return 0;
10359 }
10360
10361 int
10362 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10363                                          uint32_t port_id,
10364                                          struct rte_swx_port_out_stats *stats)
10365 {
10366         struct port_out *port;
10367
10368         if (!p || !stats)
10369                 return -EINVAL;
10370
10371         port = port_out_find(p, port_id);
10372         if (!port)
10373                 return -EINVAL;
10374
10375         port->type->ops.stats_read(port->obj, stats);
10376         return 0;
10377 }
10378
10379 int
10380 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10381                                       const char *table_name,
10382                                       struct rte_swx_table_stats *stats)
10383 {
10384         struct table *table;
10385         struct table_statistics *table_stats;
10386
10387         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10388                 return -EINVAL;
10389
10390         table = table_find(p, table_name);
10391         if (!table)
10392                 return -EINVAL;
10393
10394         table_stats = &p->table_stats[table->id];
10395
10396         memcpy(stats->n_pkts_action,
10397                table_stats->n_pkts_action,
10398                p->n_actions * sizeof(uint64_t));
10399
10400         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10401         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10402
10403         return 0;
10404 }
10405
10406 int
10407 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10408         const char *selector_name,
10409         struct rte_swx_pipeline_selector_stats *stats)
10410 {
10411         struct selector *s;
10412
10413         if (!p || !selector_name || !selector_name[0] || !stats)
10414                 return -EINVAL;
10415
10416         s = selector_find(p, selector_name);
10417         if (!s)
10418                 return -EINVAL;
10419
10420         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10421
10422         return 0;
10423 }
10424
10425 int
10426 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10427                                         const char *learner_name,
10428                                         struct rte_swx_learner_stats *stats)
10429 {
10430         struct learner *l;
10431         struct learner_statistics *learner_stats;
10432
10433         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10434                 return -EINVAL;
10435
10436         l = learner_find(p, learner_name);
10437         if (!l)
10438                 return -EINVAL;
10439
10440         learner_stats = &p->learner_stats[l->id];
10441
10442         memcpy(stats->n_pkts_action,
10443                learner_stats->n_pkts_action,
10444                p->n_actions * sizeof(uint64_t));
10445
10446         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10447         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10448
10449         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10450         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10451
10452         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10453
10454         return 0;
10455 }
10456
10457 int
10458 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10459                               uint32_t regarray_id,
10460                               struct rte_swx_ctl_regarray_info *regarray)
10461 {
10462         struct regarray *r;
10463
10464         if (!p || !regarray)
10465                 return -EINVAL;
10466
10467         r = regarray_find_by_id(p, regarray_id);
10468         if (!r)
10469                 return -EINVAL;
10470
10471         strcpy(regarray->name, r->name);
10472         regarray->size = r->size;
10473         return 0;
10474 }
10475
10476 int
10477 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10478                                    const char *regarray_name,
10479                                    uint32_t regarray_index,
10480                                    uint64_t *value)
10481 {
10482         struct regarray *regarray;
10483         struct regarray_runtime *r;
10484
10485         if (!p || !regarray_name || !value)
10486                 return -EINVAL;
10487
10488         regarray = regarray_find(p, regarray_name);
10489         if (!regarray || (regarray_index >= regarray->size))
10490                 return -EINVAL;
10491
10492         r = &p->regarray_runtime[regarray->id];
10493         *value = r->regarray[regarray_index];
10494         return 0;
10495 }
10496
10497 int
10498 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10499                                    const char *regarray_name,
10500                                    uint32_t regarray_index,
10501                                    uint64_t value)
10502 {
10503         struct regarray *regarray;
10504         struct regarray_runtime *r;
10505
10506         if (!p || !regarray_name)
10507                 return -EINVAL;
10508
10509         regarray = regarray_find(p, regarray_name);
10510         if (!regarray || (regarray_index >= regarray->size))
10511                 return -EINVAL;
10512
10513         r = &p->regarray_runtime[regarray->id];
10514         r->regarray[regarray_index] = value;
10515         return 0;
10516 }
10517
10518 int
10519 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10520                               uint32_t metarray_id,
10521                               struct rte_swx_ctl_metarray_info *metarray)
10522 {
10523         struct metarray *m;
10524
10525         if (!p || !metarray)
10526                 return -EINVAL;
10527
10528         m = metarray_find_by_id(p, metarray_id);
10529         if (!m)
10530                 return -EINVAL;
10531
10532         strcpy(metarray->name, m->name);
10533         metarray->size = m->size;
10534         return 0;
10535 }
10536
10537 int
10538 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10539                               const char *name,
10540                               struct rte_meter_trtcm_params *params)
10541 {
10542         struct meter_profile *mp;
10543         int status;
10544
10545         CHECK(p, EINVAL);
10546         CHECK_NAME(name, EINVAL);
10547         CHECK(params, EINVAL);
10548         CHECK(!meter_profile_find(p, name), EEXIST);
10549
10550         /* Node allocation. */
10551         mp = calloc(1, sizeof(struct meter_profile));
10552         CHECK(mp, ENOMEM);
10553
10554         /* Node initialization. */
10555         strcpy(mp->name, name);
10556         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10557         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10558         if (status) {
10559                 free(mp);
10560                 CHECK(0, EINVAL);
10561         }
10562
10563         /* Node add to tailq. */
10564         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10565
10566         return 0;
10567 }
10568
10569 int
10570 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10571                                  const char *name)
10572 {
10573         struct meter_profile *mp;
10574
10575         CHECK(p, EINVAL);
10576         CHECK_NAME(name, EINVAL);
10577
10578         mp = meter_profile_find(p, name);
10579         CHECK(mp, EINVAL);
10580         CHECK(!mp->n_users, EBUSY);
10581
10582         /* Remove node from tailq. */
10583         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10584         free(mp);
10585
10586         return 0;
10587 }
10588
10589 int
10590 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10591                         const char *metarray_name,
10592                         uint32_t metarray_index)
10593 {
10594         struct meter_profile *mp_old;
10595         struct metarray *metarray;
10596         struct metarray_runtime *metarray_runtime;
10597         struct meter *m;
10598
10599         CHECK(p, EINVAL);
10600         CHECK_NAME(metarray_name, EINVAL);
10601
10602         metarray = metarray_find(p, metarray_name);
10603         CHECK(metarray, EINVAL);
10604         CHECK(metarray_index < metarray->size, EINVAL);
10605
10606         metarray_runtime = &p->metarray_runtime[metarray->id];
10607         m = &metarray_runtime->metarray[metarray_index];
10608         mp_old = m->profile;
10609
10610         meter_init(m);
10611
10612         mp_old->n_users--;
10613
10614         return 0;
10615 }
10616
10617 int
10618 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10619                       const char *metarray_name,
10620                       uint32_t metarray_index,
10621                       const char *profile_name)
10622 {
10623         struct meter_profile *mp, *mp_old;
10624         struct metarray *metarray;
10625         struct metarray_runtime *metarray_runtime;
10626         struct meter *m;
10627
10628         CHECK(p, EINVAL);
10629         CHECK_NAME(metarray_name, EINVAL);
10630
10631         metarray = metarray_find(p, metarray_name);
10632         CHECK(metarray, EINVAL);
10633         CHECK(metarray_index < metarray->size, EINVAL);
10634
10635         mp = meter_profile_find(p, profile_name);
10636         CHECK(mp, EINVAL);
10637
10638         metarray_runtime = &p->metarray_runtime[metarray->id];
10639         m = &metarray_runtime->metarray[metarray_index];
10640         mp_old = m->profile;
10641
10642         memset(m, 0, sizeof(struct meter));
10643         rte_meter_trtcm_config(&m->m, &mp->profile);
10644         m->profile = mp;
10645         m->color_mask = RTE_COLORS;
10646
10647         mp->n_users++;
10648         mp_old->n_users--;
10649
10650         return 0;
10651 }
10652
10653 int
10654 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10655                              const char *metarray_name,
10656                              uint32_t metarray_index,
10657                              struct rte_swx_ctl_meter_stats *stats)
10658 {
10659         struct metarray *metarray;
10660         struct metarray_runtime *metarray_runtime;
10661         struct meter *m;
10662
10663         CHECK(p, EINVAL);
10664         CHECK_NAME(metarray_name, EINVAL);
10665
10666         metarray = metarray_find(p, metarray_name);
10667         CHECK(metarray, EINVAL);
10668         CHECK(metarray_index < metarray->size, EINVAL);
10669
10670         CHECK(stats, EINVAL);
10671
10672         metarray_runtime = &p->metarray_runtime[metarray->id];
10673         m = &metarray_runtime->metarray[metarray_index];
10674
10675         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10676         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10677
10678         return 0;
10679 }