lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_is_jmp(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_JMP:
1384         case INSTR_JMP_VALID:
1385         case INSTR_JMP_INVALID:
1386         case INSTR_JMP_HIT:
1387         case INSTR_JMP_MISS:
1388         case INSTR_JMP_ACTION_HIT:
1389         case INSTR_JMP_ACTION_MISS:
1390         case INSTR_JMP_EQ:
1391         case INSTR_JMP_EQ_MH:
1392         case INSTR_JMP_EQ_HM:
1393         case INSTR_JMP_EQ_HH:
1394         case INSTR_JMP_EQ_I:
1395         case INSTR_JMP_NEQ:
1396         case INSTR_JMP_NEQ_MH:
1397         case INSTR_JMP_NEQ_HM:
1398         case INSTR_JMP_NEQ_HH:
1399         case INSTR_JMP_NEQ_I:
1400         case INSTR_JMP_LT:
1401         case INSTR_JMP_LT_MH:
1402         case INSTR_JMP_LT_HM:
1403         case INSTR_JMP_LT_HH:
1404         case INSTR_JMP_LT_MI:
1405         case INSTR_JMP_LT_HI:
1406         case INSTR_JMP_GT:
1407         case INSTR_JMP_GT_MH:
1408         case INSTR_JMP_GT_HM:
1409         case INSTR_JMP_GT_HH:
1410         case INSTR_JMP_GT_MI:
1411         case INSTR_JMP_GT_HI:
1412                 return 1;
1413
1414         default:
1415                 return 0;
1416         }
1417 }
1418
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1421
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424                    struct action *action,
1425                    const char *name,
1426                    uint32_t *struct_id)
1427 {
1428         struct field *f;
1429
1430         switch (name[0]) {
1431         case 'h':
1432         {
1433                 struct header *header;
1434
1435                 f = header_field_parse(p, name, &header);
1436                 if (!f)
1437                         return NULL;
1438
1439                 *struct_id = header->struct_id;
1440                 return f;
1441         }
1442
1443         case 'm':
1444         {
1445                 f = metadata_field_parse(p, name);
1446                 if (!f)
1447                         return NULL;
1448
1449                 *struct_id = p->metadata_struct_id;
1450                 return f;
1451         }
1452
1453         case 't':
1454         {
1455                 if (!action)
1456                         return NULL;
1457
1458                 f = action_field_parse(action, name);
1459                 if (!f)
1460                         return NULL;
1461
1462                 *struct_id = 0;
1463                 return f;
1464         }
1465
1466         case 'e':
1467         {
1468                 struct extern_obj *obj;
1469
1470                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1471                 if (!f)
1472                         return NULL;
1473
1474                 *struct_id = obj->struct_id;
1475                 return f;
1476         }
1477
1478         case 'f':
1479         {
1480                 struct extern_func *func;
1481
1482                 f = extern_func_mailbox_field_parse(p, name, &func);
1483                 if (!f)
1484                         return NULL;
1485
1486                 *struct_id = func->struct_id;
1487                 return f;
1488         }
1489
1490         default:
1491                 return NULL;
1492         }
1493 }
1494
1495 /*
1496  * rx.
1497  */
1498 static int
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500                    struct action *action,
1501                    char **tokens,
1502                    int n_tokens,
1503                    struct instruction *instr,
1504                    struct instruction_data *data __rte_unused)
1505 {
1506         struct field *f;
1507
1508         CHECK(!action, EINVAL);
1509         CHECK(n_tokens == 2, EINVAL);
1510
1511         f = metadata_field_parse(p, tokens[1]);
1512         CHECK(f, EINVAL);
1513
1514         instr->type = INSTR_RX;
1515         instr->io.io.offset = f->offset / 8;
1516         instr->io.io.n_bits = f->n_bits;
1517         return 0;
1518 }
1519
1520 /*
1521  * tx.
1522  */
1523 static int
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525                    struct action *action __rte_unused,
1526                    char **tokens,
1527                    int n_tokens,
1528                    struct instruction *instr,
1529                    struct instruction_data *data __rte_unused)
1530 {
1531         char *port = tokens[1];
1532         struct field *f;
1533         uint32_t port_val;
1534
1535         CHECK(n_tokens == 2, EINVAL);
1536
1537         f = metadata_field_parse(p, port);
1538         if (f) {
1539                 instr->type = INSTR_TX;
1540                 instr->io.io.offset = f->offset / 8;
1541                 instr->io.io.n_bits = f->n_bits;
1542                 return 0;
1543         }
1544
1545         /* TX_I. */
1546         port_val = strtoul(port, &port, 0);
1547         CHECK(!port[0], EINVAL);
1548
1549         instr->type = INSTR_TX_I;
1550         instr->io.io.val = port_val;
1551         return 0;
1552 }
1553
1554 static int
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556                      struct action *action __rte_unused,
1557                      char **tokens __rte_unused,
1558                      int n_tokens,
1559                      struct instruction *instr,
1560                      struct instruction_data *data __rte_unused)
1561 {
1562         CHECK(n_tokens == 1, EINVAL);
1563
1564         /* TX_I. */
1565         instr->type = INSTR_TX_I;
1566         instr->io.io.val = p->n_ports_out - 1;
1567         return 0;
1568 }
1569
1570 static inline void
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1572 {
1573         struct thread *t = &p->threads[p->thread_id];
1574         struct instruction *ip = t->ip;
1575
1576         __instr_tx_exec(p, t, ip);
1577
1578         /* Thread. */
1579         thread_ip_reset(p, t);
1580         instr_rx_exec(p);
1581 }
1582
1583 static inline void
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1585 {
1586         struct thread *t = &p->threads[p->thread_id];
1587         struct instruction *ip = t->ip;
1588
1589         __instr_tx_i_exec(p, t, ip);
1590
1591         /* Thread. */
1592         thread_ip_reset(p, t);
1593         instr_rx_exec(p);
1594 }
1595
1596 /*
1597  * extract.
1598  */
1599 static int
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601                             struct action *action,
1602                             char **tokens,
1603                             int n_tokens,
1604                             struct instruction *instr,
1605                             struct instruction_data *data __rte_unused)
1606 {
1607         struct header *h;
1608
1609         CHECK(!action, EINVAL);
1610         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1611
1612         h = header_parse(p, tokens[1]);
1613         CHECK(h, EINVAL);
1614
1615         if (n_tokens == 2) {
1616                 CHECK(!h->st->var_size, EINVAL);
1617
1618                 instr->type = INSTR_HDR_EXTRACT;
1619                 instr->io.hdr.header_id[0] = h->id;
1620                 instr->io.hdr.struct_id[0] = h->struct_id;
1621                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1622         } else {
1623                 struct field *mf;
1624
1625                 CHECK(h->st->var_size, EINVAL);
1626
1627                 mf = metadata_field_parse(p, tokens[2]);
1628                 CHECK(mf, EINVAL);
1629                 CHECK(!mf->var_size, EINVAL);
1630
1631                 instr->type = INSTR_HDR_EXTRACT_M;
1632                 instr->io.io.offset = mf->offset / 8;
1633                 instr->io.io.n_bits = mf->n_bits;
1634                 instr->io.hdr.header_id[0] = h->id;
1635                 instr->io.hdr.struct_id[0] = h->struct_id;
1636                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644                               struct action *action,
1645                               char **tokens,
1646                               int n_tokens,
1647                               struct instruction *instr,
1648                               struct instruction_data *data __rte_unused)
1649 {
1650         struct header *h;
1651
1652         CHECK(!action, EINVAL);
1653         CHECK(n_tokens == 2, EINVAL);
1654
1655         h = header_parse(p, tokens[1]);
1656         CHECK(h, EINVAL);
1657         CHECK(!h->st->var_size, EINVAL);
1658
1659         instr->type = INSTR_HDR_LOOKAHEAD;
1660         instr->io.hdr.header_id[0] = h->id;
1661         instr->io.hdr.struct_id[0] = h->struct_id;
1662         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1663
1664         return 0;
1665 }
1666
1667 static inline void
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1669 {
1670         struct thread *t = &p->threads[p->thread_id];
1671         struct instruction *ip = t->ip;
1672
1673         __instr_hdr_extract_exec(p, t, ip);
1674
1675         /* Thread. */
1676         thread_ip_inc(p);
1677 }
1678
1679 static inline void
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1681 {
1682         struct thread *t = &p->threads[p->thread_id];
1683         struct instruction *ip = t->ip;
1684
1685         __instr_hdr_extract2_exec(p, t, ip);
1686
1687         /* Thread. */
1688         thread_ip_inc(p);
1689 }
1690
1691 static inline void
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1693 {
1694         struct thread *t = &p->threads[p->thread_id];
1695         struct instruction *ip = t->ip;
1696
1697         __instr_hdr_extract3_exec(p, t, ip);
1698
1699         /* Thread. */
1700         thread_ip_inc(p);
1701 }
1702
1703 static inline void
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1705 {
1706         struct thread *t = &p->threads[p->thread_id];
1707         struct instruction *ip = t->ip;
1708
1709         __instr_hdr_extract4_exec(p, t, ip);
1710
1711         /* Thread. */
1712         thread_ip_inc(p);
1713 }
1714
1715 static inline void
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1717 {
1718         struct thread *t = &p->threads[p->thread_id];
1719         struct instruction *ip = t->ip;
1720
1721         __instr_hdr_extract5_exec(p, t, ip);
1722
1723         /* Thread. */
1724         thread_ip_inc(p);
1725 }
1726
1727 static inline void
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract6_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract7_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract8_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract_m_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_lookahead_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 /*
1788  * emit.
1789  */
1790 static int
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792                          struct action *action __rte_unused,
1793                          char **tokens,
1794                          int n_tokens,
1795                          struct instruction *instr,
1796                          struct instruction_data *data __rte_unused)
1797 {
1798         struct header *h;
1799
1800         CHECK(n_tokens == 2, EINVAL);
1801
1802         h = header_parse(p, tokens[1]);
1803         CHECK(h, EINVAL);
1804
1805         instr->type = INSTR_HDR_EMIT;
1806         instr->io.hdr.header_id[0] = h->id;
1807         instr->io.hdr.struct_id[0] = h->struct_id;
1808         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1809         return 0;
1810 }
1811
1812 static inline void
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_emit_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_emit_tx_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_reset(p, t);
1834         instr_rx_exec(p);
1835 }
1836
1837 static inline void
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1839 {
1840         struct thread *t = &p->threads[p->thread_id];
1841         struct instruction *ip = t->ip;
1842
1843         __instr_hdr_emit2_tx_exec(p, t, ip);
1844
1845         /* Thread. */
1846         thread_ip_reset(p, t);
1847         instr_rx_exec(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854         struct instruction *ip = t->ip;
1855
1856         __instr_hdr_emit3_tx_exec(p, t, ip);
1857
1858         /* Thread. */
1859         thread_ip_reset(p, t);
1860         instr_rx_exec(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit4_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit5_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit6_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit7_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit8_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 /*
1929  * validate.
1930  */
1931 static int
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933                              struct action *action __rte_unused,
1934                              char **tokens,
1935                              int n_tokens,
1936                              struct instruction *instr,
1937                              struct instruction_data *data __rte_unused)
1938 {
1939         struct header *h;
1940
1941         CHECK(n_tokens == 2, EINVAL);
1942
1943         h = header_parse(p, tokens[1]);
1944         CHECK(h, EINVAL);
1945
1946         instr->type = INSTR_HDR_VALIDATE;
1947         instr->valid.header_id = h->id;
1948         return 0;
1949 }
1950
1951 static inline void
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1953 {
1954         struct thread *t = &p->threads[p->thread_id];
1955         struct instruction *ip = t->ip;
1956
1957         __instr_hdr_validate_exec(p, t, ip);
1958
1959         /* Thread. */
1960         thread_ip_inc(p);
1961 }
1962
1963 /*
1964  * invalidate.
1965  */
1966 static int
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968                                struct action *action __rte_unused,
1969                                char **tokens,
1970                                int n_tokens,
1971                                struct instruction *instr,
1972                                struct instruction_data *data __rte_unused)
1973 {
1974         struct header *h;
1975
1976         CHECK(n_tokens == 2, EINVAL);
1977
1978         h = header_parse(p, tokens[1]);
1979         CHECK(h, EINVAL);
1980
1981         instr->type = INSTR_HDR_INVALIDATE;
1982         instr->valid.header_id = h->id;
1983         return 0;
1984 }
1985
1986 static inline void
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1988 {
1989         struct thread *t = &p->threads[p->thread_id];
1990         struct instruction *ip = t->ip;
1991
1992         __instr_hdr_invalidate_exec(p, t, ip);
1993
1994         /* Thread. */
1995         thread_ip_inc(p);
1996 }
1997
1998 /*
1999  * table.
2000  */
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2003
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2006
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2009
2010 static int
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012                       struct action *action,
2013                       char **tokens,
2014                       int n_tokens,
2015                       struct instruction *instr,
2016                       struct instruction_data *data __rte_unused)
2017 {
2018         struct table *t;
2019         struct selector *s;
2020         struct learner *l;
2021
2022         CHECK(!action, EINVAL);
2023         CHECK(n_tokens == 2, EINVAL);
2024
2025         t = table_find(p, tokens[1]);
2026         if (t) {
2027                 instr->type = INSTR_TABLE;
2028                 instr->table.table_id = t->id;
2029                 return 0;
2030         }
2031
2032         s = selector_find(p, tokens[1]);
2033         if (s) {
2034                 instr->type = INSTR_SELECTOR;
2035                 instr->table.table_id = s->id;
2036                 return 0;
2037         }
2038
2039         l = learner_find(p, tokens[1]);
2040         if (l) {
2041                 instr->type = INSTR_LEARNER;
2042                 instr->table.table_id = l->id;
2043                 return 0;
2044         }
2045
2046         CHECK(0, EINVAL);
2047 }
2048
2049 static inline void
2050 instr_table_exec(struct rte_swx_pipeline *p)
2051 {
2052         struct thread *t = &p->threads[p->thread_id];
2053         struct instruction *ip = t->ip;
2054         uint32_t table_id = ip->table.table_id;
2055         struct rte_swx_table_state *ts = &t->table_state[table_id];
2056         struct table_runtime *table = &t->tables[table_id];
2057         struct table_statistics *stats = &p->table_stats[table_id];
2058         uint64_t action_id, n_pkts_hit, n_pkts_action;
2059         uint8_t *action_data;
2060         int done, hit;
2061
2062         /* Table. */
2063         done = table->func(ts->obj,
2064                            table->mailbox,
2065                            table->key,
2066                            &action_id,
2067                            &action_data,
2068                            &hit);
2069         if (!done) {
2070                 /* Thread. */
2071                 TRACE("[Thread %2u] table %u (not finalized)\n",
2072                       p->thread_id,
2073                       table_id);
2074
2075                 thread_yield(p);
2076                 return;
2077         }
2078
2079         action_id = hit ? action_id : ts->default_action_id;
2080         action_data = hit ? action_data : ts->default_action_data;
2081         n_pkts_hit = stats->n_pkts_hit[hit];
2082         n_pkts_action = stats->n_pkts_action[action_id];
2083
2084         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2085               p->thread_id,
2086               table_id,
2087               hit ? "hit" : "miss",
2088               (uint32_t)action_id);
2089
2090         t->action_id = action_id;
2091         t->structs[0] = action_data;
2092         t->hit = hit;
2093         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2095
2096         /* Thread. */
2097         thread_ip_action_call(p, t, action_id);
2098 }
2099
2100 static inline void
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         uint32_t selector_id = ip->table.table_id;
2106         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107         struct selector_runtime *selector = &t->selectors[selector_id];
2108         struct selector_statistics *stats = &p->selector_stats[selector_id];
2109         uint64_t n_pkts = stats->n_pkts;
2110         int done;
2111
2112         /* Table. */
2113         done = rte_swx_table_selector_select(ts->obj,
2114                            selector->mailbox,
2115                            selector->group_id_buffer,
2116                            selector->selector_buffer,
2117                            selector->member_id_buffer);
2118         if (!done) {
2119                 /* Thread. */
2120                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2121                       p->thread_id,
2122                       selector_id);
2123
2124                 thread_yield(p);
2125                 return;
2126         }
2127
2128
2129         TRACE("[Thread %2u] selector %u\n",
2130               p->thread_id,
2131               selector_id);
2132
2133         stats->n_pkts = n_pkts + 1;
2134
2135         /* Thread. */
2136         thread_ip_inc(p);
2137 }
2138
2139 static inline void
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t learner_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146                 p->n_selectors + learner_id];
2147         struct learner_runtime *l = &t->learners[learner_id];
2148         struct learner_statistics *stats = &p->learner_stats[learner_id];
2149         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150         uint8_t *action_data;
2151         int done, hit;
2152
2153         /* Table. */
2154         time = rte_get_tsc_cycles();
2155
2156         done = rte_swx_table_learner_lookup(ts->obj,
2157                                             l->mailbox,
2158                                             time,
2159                                             l->key,
2160                                             &action_id,
2161                                             &action_data,
2162                                             &hit);
2163         if (!done) {
2164                 /* Thread. */
2165                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2166                       p->thread_id,
2167                       learner_id);
2168
2169                 thread_yield(p);
2170                 return;
2171         }
2172
2173         action_id = hit ? action_id : ts->default_action_id;
2174         action_data = hit ? action_data : ts->default_action_data;
2175         n_pkts_hit = stats->n_pkts_hit[hit];
2176         n_pkts_action = stats->n_pkts_action[action_id];
2177
2178         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2179               p->thread_id,
2180               learner_id,
2181               hit ? "hit" : "miss",
2182               (uint32_t)action_id);
2183
2184         t->action_id = action_id;
2185         t->structs[0] = action_data;
2186         t->hit = hit;
2187         t->learner_id = learner_id;
2188         t->time = time;
2189         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2191
2192         /* Thread. */
2193         thread_ip_action_call(p, t, action_id);
2194 }
2195
2196 /*
2197  * learn.
2198  */
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2201
2202 static int
2203 action_has_nbo_args(struct action *a);
2204
2205 static int
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207                       struct action *action,
2208                       char **tokens,
2209                       int n_tokens,
2210                       struct instruction *instr,
2211                       struct instruction_data *data __rte_unused)
2212 {
2213         struct action *a;
2214
2215         CHECK(action, EINVAL);
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         a = action_find(p, tokens[1]);
2219         CHECK(a, EINVAL);
2220         CHECK(!action_has_nbo_args(a), EINVAL);
2221
2222         instr->type = INSTR_LEARNER_LEARN;
2223         instr->learn.action_id = a->id;
2224
2225         return 0;
2226 }
2227
2228 static inline void
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_learn_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_inc(p);
2238 }
2239
2240 /*
2241  * forget.
2242  */
2243 static int
2244 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2245                        struct action *action,
2246                        char **tokens __rte_unused,
2247                        int n_tokens,
2248                        struct instruction *instr,
2249                        struct instruction_data *data __rte_unused)
2250 {
2251         CHECK(action, EINVAL);
2252         CHECK(n_tokens == 1, EINVAL);
2253
2254         instr->type = INSTR_LEARNER_FORGET;
2255
2256         return 0;
2257 }
2258
2259 static inline void
2260 instr_forget_exec(struct rte_swx_pipeline *p)
2261 {
2262         struct thread *t = &p->threads[p->thread_id];
2263         struct instruction *ip = t->ip;
2264
2265         __instr_forget_exec(p, t, ip);
2266
2267         /* Thread. */
2268         thread_ip_inc(p);
2269 }
2270
2271 /*
2272  * extern.
2273  */
2274 static int
2275 instr_extern_translate(struct rte_swx_pipeline *p,
2276                        struct action *action __rte_unused,
2277                        char **tokens,
2278                        int n_tokens,
2279                        struct instruction *instr,
2280                        struct instruction_data *data __rte_unused)
2281 {
2282         char *token = tokens[1];
2283
2284         CHECK(n_tokens == 2, EINVAL);
2285
2286         if (token[0] == 'e') {
2287                 struct extern_obj *obj;
2288                 struct extern_type_member_func *func;
2289
2290                 func = extern_obj_member_func_parse(p, token, &obj);
2291                 CHECK(func, EINVAL);
2292
2293                 instr->type = INSTR_EXTERN_OBJ;
2294                 instr->ext_obj.ext_obj_id = obj->id;
2295                 instr->ext_obj.func_id = func->id;
2296
2297                 return 0;
2298         }
2299
2300         if (token[0] == 'f') {
2301                 struct extern_func *func;
2302
2303                 func = extern_func_parse(p, token);
2304                 CHECK(func, EINVAL);
2305
2306                 instr->type = INSTR_EXTERN_FUNC;
2307                 instr->ext_func.ext_func_id = func->id;
2308
2309                 return 0;
2310         }
2311
2312         CHECK(0, EINVAL);
2313 }
2314
2315 static inline void
2316 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2317 {
2318         struct thread *t = &p->threads[p->thread_id];
2319         struct instruction *ip = t->ip;
2320         uint32_t done;
2321
2322         /* Extern object member function execute. */
2323         done = __instr_extern_obj_exec(p, t, ip);
2324
2325         /* Thread. */
2326         thread_ip_inc_cond(t, done);
2327         thread_yield_cond(p, done ^ 1);
2328 }
2329
2330 static inline void
2331 instr_extern_func_exec(struct rte_swx_pipeline *p)
2332 {
2333         struct thread *t = &p->threads[p->thread_id];
2334         struct instruction *ip = t->ip;
2335         uint32_t done;
2336
2337         /* Extern function execute. */
2338         done = __instr_extern_func_exec(p, t, ip);
2339
2340         /* Thread. */
2341         thread_ip_inc_cond(t, done);
2342         thread_yield_cond(p, done ^ 1);
2343 }
2344
2345 /*
2346  * mov.
2347  */
2348 static int
2349 instr_mov_translate(struct rte_swx_pipeline *p,
2350                     struct action *action,
2351                     char **tokens,
2352                     int n_tokens,
2353                     struct instruction *instr,
2354                     struct instruction_data *data __rte_unused)
2355 {
2356         char *dst = tokens[1], *src = tokens[2];
2357         struct field *fdst, *fsrc;
2358         uint64_t src_val;
2359         uint32_t dst_struct_id = 0, src_struct_id = 0;
2360
2361         CHECK(n_tokens == 3, EINVAL);
2362
2363         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2364         CHECK(fdst, EINVAL);
2365         CHECK(!fdst->var_size, EINVAL);
2366
2367         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2368         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2369         if (fsrc) {
2370                 CHECK(!fsrc->var_size, EINVAL);
2371
2372                 instr->type = INSTR_MOV;
2373                 if (dst[0] != 'h' && src[0] == 'h')
2374                         instr->type = INSTR_MOV_MH;
2375                 if (dst[0] == 'h' && src[0] != 'h')
2376                         instr->type = INSTR_MOV_HM;
2377                 if (dst[0] == 'h' && src[0] == 'h')
2378                         instr->type = INSTR_MOV_HH;
2379
2380                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2381                 instr->mov.dst.n_bits = fdst->n_bits;
2382                 instr->mov.dst.offset = fdst->offset / 8;
2383                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2384                 instr->mov.src.n_bits = fsrc->n_bits;
2385                 instr->mov.src.offset = fsrc->offset / 8;
2386                 return 0;
2387         }
2388
2389         /* MOV_I. */
2390         src_val = strtoull(src, &src, 0);
2391         CHECK(!src[0], EINVAL);
2392
2393         if (dst[0] == 'h')
2394                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2395
2396         instr->type = INSTR_MOV_I;
2397         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2398         instr->mov.dst.n_bits = fdst->n_bits;
2399         instr->mov.dst.offset = fdst->offset / 8;
2400         instr->mov.src_val = src_val;
2401         return 0;
2402 }
2403
2404 static inline void
2405 instr_mov_exec(struct rte_swx_pipeline *p)
2406 {
2407         struct thread *t = &p->threads[p->thread_id];
2408         struct instruction *ip = t->ip;
2409
2410         __instr_mov_exec(p, t, ip);
2411
2412         /* Thread. */
2413         thread_ip_inc(p);
2414 }
2415
2416 static inline void
2417 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2418 {
2419         struct thread *t = &p->threads[p->thread_id];
2420         struct instruction *ip = t->ip;
2421
2422         __instr_mov_mh_exec(p, t, ip);
2423
2424         /* Thread. */
2425         thread_ip_inc(p);
2426 }
2427
2428 static inline void
2429 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2430 {
2431         struct thread *t = &p->threads[p->thread_id];
2432         struct instruction *ip = t->ip;
2433
2434         __instr_mov_hm_exec(p, t, ip);
2435
2436         /* Thread. */
2437         thread_ip_inc(p);
2438 }
2439
2440 static inline void
2441 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2442 {
2443         struct thread *t = &p->threads[p->thread_id];
2444         struct instruction *ip = t->ip;
2445
2446         __instr_mov_hh_exec(p, t, ip);
2447
2448         /* Thread. */
2449         thread_ip_inc(p);
2450 }
2451
2452 static inline void
2453 instr_mov_i_exec(struct rte_swx_pipeline *p)
2454 {
2455         struct thread *t = &p->threads[p->thread_id];
2456         struct instruction *ip = t->ip;
2457
2458         __instr_mov_i_exec(p, t, ip);
2459
2460         /* Thread. */
2461         thread_ip_inc(p);
2462 }
2463
2464 /*
2465  * dma.
2466  */
2467 static inline void
2468 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2469 {
2470         struct thread *t = &p->threads[p->thread_id];
2471         struct instruction *ip = t->ip;
2472
2473         __instr_dma_ht_exec(p, t, ip);
2474
2475         /* Thread. */
2476         thread_ip_inc(p);
2477 }
2478
2479 static inline void
2480 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2481 {
2482         struct thread *t = &p->threads[p->thread_id];
2483         struct instruction *ip = t->ip;
2484
2485         __instr_dma_ht2_exec(p, t, ip);
2486
2487         /* Thread. */
2488         thread_ip_inc(p);
2489 }
2490
2491 static inline void
2492 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2493 {
2494         struct thread *t = &p->threads[p->thread_id];
2495         struct instruction *ip = t->ip;
2496
2497         __instr_dma_ht3_exec(p, t, ip);
2498
2499         /* Thread. */
2500         thread_ip_inc(p);
2501 }
2502
2503 static inline void
2504 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2505 {
2506         struct thread *t = &p->threads[p->thread_id];
2507         struct instruction *ip = t->ip;
2508
2509         __instr_dma_ht4_exec(p, t, ip);
2510
2511         /* Thread. */
2512         thread_ip_inc(p);
2513 }
2514
2515 static inline void
2516 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2517 {
2518         struct thread *t = &p->threads[p->thread_id];
2519         struct instruction *ip = t->ip;
2520
2521         __instr_dma_ht5_exec(p, t, ip);
2522
2523         /* Thread. */
2524         thread_ip_inc(p);
2525 }
2526
2527 static inline void
2528 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2529 {
2530         struct thread *t = &p->threads[p->thread_id];
2531         struct instruction *ip = t->ip;
2532
2533         __instr_dma_ht6_exec(p, t, ip);
2534
2535         /* Thread. */
2536         thread_ip_inc(p);
2537 }
2538
2539 static inline void
2540 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2541 {
2542         struct thread *t = &p->threads[p->thread_id];
2543         struct instruction *ip = t->ip;
2544
2545         __instr_dma_ht7_exec(p, t, ip);
2546
2547         /* Thread. */
2548         thread_ip_inc(p);
2549 }
2550
2551 static inline void
2552 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2553 {
2554         struct thread *t = &p->threads[p->thread_id];
2555         struct instruction *ip = t->ip;
2556
2557         __instr_dma_ht8_exec(p, t, ip);
2558
2559         /* Thread. */
2560         thread_ip_inc(p);
2561 }
2562
2563 /*
2564  * alu.
2565  */
2566 static int
2567 instr_alu_add_translate(struct rte_swx_pipeline *p,
2568                         struct action *action,
2569                         char **tokens,
2570                         int n_tokens,
2571                         struct instruction *instr,
2572                         struct instruction_data *data __rte_unused)
2573 {
2574         char *dst = tokens[1], *src = tokens[2];
2575         struct field *fdst, *fsrc;
2576         uint64_t src_val;
2577         uint32_t dst_struct_id = 0, src_struct_id = 0;
2578
2579         CHECK(n_tokens == 3, EINVAL);
2580
2581         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2582         CHECK(fdst, EINVAL);
2583         CHECK(!fdst->var_size, EINVAL);
2584
2585         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2586         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2587         if (fsrc) {
2588                 CHECK(!fsrc->var_size, EINVAL);
2589
2590                 instr->type = INSTR_ALU_ADD;
2591                 if (dst[0] == 'h' && src[0] != 'h')
2592                         instr->type = INSTR_ALU_ADD_HM;
2593                 if (dst[0] != 'h' && src[0] == 'h')
2594                         instr->type = INSTR_ALU_ADD_MH;
2595                 if (dst[0] == 'h' && src[0] == 'h')
2596                         instr->type = INSTR_ALU_ADD_HH;
2597
2598                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2599                 instr->alu.dst.n_bits = fdst->n_bits;
2600                 instr->alu.dst.offset = fdst->offset / 8;
2601                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2602                 instr->alu.src.n_bits = fsrc->n_bits;
2603                 instr->alu.src.offset = fsrc->offset / 8;
2604                 return 0;
2605         }
2606
2607         /* ADD_MI, ADD_HI. */
2608         src_val = strtoull(src, &src, 0);
2609         CHECK(!src[0], EINVAL);
2610
2611         instr->type = INSTR_ALU_ADD_MI;
2612         if (dst[0] == 'h')
2613                 instr->type = INSTR_ALU_ADD_HI;
2614
2615         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2616         instr->alu.dst.n_bits = fdst->n_bits;
2617         instr->alu.dst.offset = fdst->offset / 8;
2618         instr->alu.src_val = src_val;
2619         return 0;
2620 }
2621
2622 static int
2623 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2624                         struct action *action,
2625                         char **tokens,
2626                         int n_tokens,
2627                         struct instruction *instr,
2628                         struct instruction_data *data __rte_unused)
2629 {
2630         char *dst = tokens[1], *src = tokens[2];
2631         struct field *fdst, *fsrc;
2632         uint64_t src_val;
2633         uint32_t dst_struct_id = 0, src_struct_id = 0;
2634
2635         CHECK(n_tokens == 3, EINVAL);
2636
2637         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2638         CHECK(fdst, EINVAL);
2639         CHECK(!fdst->var_size, EINVAL);
2640
2641         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2642         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2643         if (fsrc) {
2644                 CHECK(!fsrc->var_size, EINVAL);
2645
2646                 instr->type = INSTR_ALU_SUB;
2647                 if (dst[0] == 'h' && src[0] != 'h')
2648                         instr->type = INSTR_ALU_SUB_HM;
2649                 if (dst[0] != 'h' && src[0] == 'h')
2650                         instr->type = INSTR_ALU_SUB_MH;
2651                 if (dst[0] == 'h' && src[0] == 'h')
2652                         instr->type = INSTR_ALU_SUB_HH;
2653
2654                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2655                 instr->alu.dst.n_bits = fdst->n_bits;
2656                 instr->alu.dst.offset = fdst->offset / 8;
2657                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2658                 instr->alu.src.n_bits = fsrc->n_bits;
2659                 instr->alu.src.offset = fsrc->offset / 8;
2660                 return 0;
2661         }
2662
2663         /* SUB_MI, SUB_HI. */
2664         src_val = strtoull(src, &src, 0);
2665         CHECK(!src[0], EINVAL);
2666
2667         instr->type = INSTR_ALU_SUB_MI;
2668         if (dst[0] == 'h')
2669                 instr->type = INSTR_ALU_SUB_HI;
2670
2671         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2672         instr->alu.dst.n_bits = fdst->n_bits;
2673         instr->alu.dst.offset = fdst->offset / 8;
2674         instr->alu.src_val = src_val;
2675         return 0;
2676 }
2677
2678 static int
2679 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2680                           struct action *action __rte_unused,
2681                           char **tokens,
2682                           int n_tokens,
2683                           struct instruction *instr,
2684                           struct instruction_data *data __rte_unused)
2685 {
2686         char *dst = tokens[1], *src = tokens[2];
2687         struct header *hdst, *hsrc;
2688         struct field *fdst, *fsrc;
2689
2690         CHECK(n_tokens == 3, EINVAL);
2691
2692         fdst = header_field_parse(p, dst, &hdst);
2693         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2694         CHECK(!fdst->var_size, EINVAL);
2695
2696         /* CKADD_FIELD. */
2697         fsrc = header_field_parse(p, src, &hsrc);
2698         if (fsrc) {
2699                 CHECK(!fsrc->var_size, EINVAL);
2700
2701                 instr->type = INSTR_ALU_CKADD_FIELD;
2702                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2703                 instr->alu.dst.n_bits = fdst->n_bits;
2704                 instr->alu.dst.offset = fdst->offset / 8;
2705                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2706                 instr->alu.src.n_bits = fsrc->n_bits;
2707                 instr->alu.src.offset = fsrc->offset / 8;
2708                 return 0;
2709         }
2710
2711         /* CKADD_STRUCT, CKADD_STRUCT20. */
2712         hsrc = header_parse(p, src);
2713         CHECK(hsrc, EINVAL);
2714         CHECK(!hsrc->st->var_size, EINVAL);
2715
2716         instr->type = INSTR_ALU_CKADD_STRUCT;
2717         if ((hsrc->st->n_bits / 8) == 20)
2718                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2719
2720         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2721         instr->alu.dst.n_bits = fdst->n_bits;
2722         instr->alu.dst.offset = fdst->offset / 8;
2723         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2724         instr->alu.src.n_bits = hsrc->st->n_bits;
2725         instr->alu.src.offset = 0; /* Unused. */
2726         return 0;
2727 }
2728
2729 static int
2730 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2731                           struct action *action __rte_unused,
2732                           char **tokens,
2733                           int n_tokens,
2734                           struct instruction *instr,
2735                           struct instruction_data *data __rte_unused)
2736 {
2737         char *dst = tokens[1], *src = tokens[2];
2738         struct header *hdst, *hsrc;
2739         struct field *fdst, *fsrc;
2740
2741         CHECK(n_tokens == 3, EINVAL);
2742
2743         fdst = header_field_parse(p, dst, &hdst);
2744         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2745         CHECK(!fdst->var_size, EINVAL);
2746
2747         fsrc = header_field_parse(p, src, &hsrc);
2748         CHECK(fsrc, EINVAL);
2749         CHECK(!fsrc->var_size, EINVAL);
2750
2751         instr->type = INSTR_ALU_CKSUB_FIELD;
2752         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2753         instr->alu.dst.n_bits = fdst->n_bits;
2754         instr->alu.dst.offset = fdst->offset / 8;
2755         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2756         instr->alu.src.n_bits = fsrc->n_bits;
2757         instr->alu.src.offset = fsrc->offset / 8;
2758         return 0;
2759 }
2760
2761 static int
2762 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2763                         struct action *action,
2764                         char **tokens,
2765                         int n_tokens,
2766                         struct instruction *instr,
2767                         struct instruction_data *data __rte_unused)
2768 {
2769         char *dst = tokens[1], *src = tokens[2];
2770         struct field *fdst, *fsrc;
2771         uint64_t src_val;
2772         uint32_t dst_struct_id = 0, src_struct_id = 0;
2773
2774         CHECK(n_tokens == 3, EINVAL);
2775
2776         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2777         CHECK(fdst, EINVAL);
2778         CHECK(!fdst->var_size, EINVAL);
2779
2780         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2781         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2782         if (fsrc) {
2783                 CHECK(!fsrc->var_size, EINVAL);
2784
2785                 instr->type = INSTR_ALU_SHL;
2786                 if (dst[0] == 'h' && src[0] != 'h')
2787                         instr->type = INSTR_ALU_SHL_HM;
2788                 if (dst[0] != 'h' && src[0] == 'h')
2789                         instr->type = INSTR_ALU_SHL_MH;
2790                 if (dst[0] == 'h' && src[0] == 'h')
2791                         instr->type = INSTR_ALU_SHL_HH;
2792
2793                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2794                 instr->alu.dst.n_bits = fdst->n_bits;
2795                 instr->alu.dst.offset = fdst->offset / 8;
2796                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2797                 instr->alu.src.n_bits = fsrc->n_bits;
2798                 instr->alu.src.offset = fsrc->offset / 8;
2799                 return 0;
2800         }
2801
2802         /* SHL_MI, SHL_HI. */
2803         src_val = strtoull(src, &src, 0);
2804         CHECK(!src[0], EINVAL);
2805
2806         instr->type = INSTR_ALU_SHL_MI;
2807         if (dst[0] == 'h')
2808                 instr->type = INSTR_ALU_SHL_HI;
2809
2810         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2811         instr->alu.dst.n_bits = fdst->n_bits;
2812         instr->alu.dst.offset = fdst->offset / 8;
2813         instr->alu.src_val = src_val;
2814         return 0;
2815 }
2816
2817 static int
2818 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2819                         struct action *action,
2820                         char **tokens,
2821                         int n_tokens,
2822                         struct instruction *instr,
2823                         struct instruction_data *data __rte_unused)
2824 {
2825         char *dst = tokens[1], *src = tokens[2];
2826         struct field *fdst, *fsrc;
2827         uint64_t src_val;
2828         uint32_t dst_struct_id = 0, src_struct_id = 0;
2829
2830         CHECK(n_tokens == 3, EINVAL);
2831
2832         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2833         CHECK(fdst, EINVAL);
2834         CHECK(!fdst->var_size, EINVAL);
2835
2836         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2837         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2838         if (fsrc) {
2839                 CHECK(!fsrc->var_size, EINVAL);
2840
2841                 instr->type = INSTR_ALU_SHR;
2842                 if (dst[0] == 'h' && src[0] != 'h')
2843                         instr->type = INSTR_ALU_SHR_HM;
2844                 if (dst[0] != 'h' && src[0] == 'h')
2845                         instr->type = INSTR_ALU_SHR_MH;
2846                 if (dst[0] == 'h' && src[0] == 'h')
2847                         instr->type = INSTR_ALU_SHR_HH;
2848
2849                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2850                 instr->alu.dst.n_bits = fdst->n_bits;
2851                 instr->alu.dst.offset = fdst->offset / 8;
2852                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2853                 instr->alu.src.n_bits = fsrc->n_bits;
2854                 instr->alu.src.offset = fsrc->offset / 8;
2855                 return 0;
2856         }
2857
2858         /* SHR_MI, SHR_HI. */
2859         src_val = strtoull(src, &src, 0);
2860         CHECK(!src[0], EINVAL);
2861
2862         instr->type = INSTR_ALU_SHR_MI;
2863         if (dst[0] == 'h')
2864                 instr->type = INSTR_ALU_SHR_HI;
2865
2866         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2867         instr->alu.dst.n_bits = fdst->n_bits;
2868         instr->alu.dst.offset = fdst->offset / 8;
2869         instr->alu.src_val = src_val;
2870         return 0;
2871 }
2872
2873 static int
2874 instr_alu_and_translate(struct rte_swx_pipeline *p,
2875                         struct action *action,
2876                         char **tokens,
2877                         int n_tokens,
2878                         struct instruction *instr,
2879                         struct instruction_data *data __rte_unused)
2880 {
2881         char *dst = tokens[1], *src = tokens[2];
2882         struct field *fdst, *fsrc;
2883         uint64_t src_val;
2884         uint32_t dst_struct_id = 0, src_struct_id = 0;
2885
2886         CHECK(n_tokens == 3, EINVAL);
2887
2888         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2889         CHECK(fdst, EINVAL);
2890         CHECK(!fdst->var_size, EINVAL);
2891
2892         /* AND, AND_MH, AND_HM, AND_HH. */
2893         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2894         if (fsrc) {
2895                 CHECK(!fsrc->var_size, EINVAL);
2896
2897                 instr->type = INSTR_ALU_AND;
2898                 if (dst[0] != 'h' && src[0] == 'h')
2899                         instr->type = INSTR_ALU_AND_MH;
2900                 if (dst[0] == 'h' && src[0] != 'h')
2901                         instr->type = INSTR_ALU_AND_HM;
2902                 if (dst[0] == 'h' && src[0] == 'h')
2903                         instr->type = INSTR_ALU_AND_HH;
2904
2905                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2906                 instr->alu.dst.n_bits = fdst->n_bits;
2907                 instr->alu.dst.offset = fdst->offset / 8;
2908                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2909                 instr->alu.src.n_bits = fsrc->n_bits;
2910                 instr->alu.src.offset = fsrc->offset / 8;
2911                 return 0;
2912         }
2913
2914         /* AND_I. */
2915         src_val = strtoull(src, &src, 0);
2916         CHECK(!src[0], EINVAL);
2917
2918         if (dst[0] == 'h')
2919                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2920
2921         instr->type = INSTR_ALU_AND_I;
2922         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2923         instr->alu.dst.n_bits = fdst->n_bits;
2924         instr->alu.dst.offset = fdst->offset / 8;
2925         instr->alu.src_val = src_val;
2926         return 0;
2927 }
2928
2929 static int
2930 instr_alu_or_translate(struct rte_swx_pipeline *p,
2931                        struct action *action,
2932                        char **tokens,
2933                        int n_tokens,
2934                        struct instruction *instr,
2935                        struct instruction_data *data __rte_unused)
2936 {
2937         char *dst = tokens[1], *src = tokens[2];
2938         struct field *fdst, *fsrc;
2939         uint64_t src_val;
2940         uint32_t dst_struct_id = 0, src_struct_id = 0;
2941
2942         CHECK(n_tokens == 3, EINVAL);
2943
2944         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2945         CHECK(fdst, EINVAL);
2946         CHECK(!fdst->var_size, EINVAL);
2947
2948         /* OR, OR_MH, OR_HM, OR_HH. */
2949         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2950         if (fsrc) {
2951                 CHECK(!fsrc->var_size, EINVAL);
2952
2953                 instr->type = INSTR_ALU_OR;
2954                 if (dst[0] != 'h' && src[0] == 'h')
2955                         instr->type = INSTR_ALU_OR_MH;
2956                 if (dst[0] == 'h' && src[0] != 'h')
2957                         instr->type = INSTR_ALU_OR_HM;
2958                 if (dst[0] == 'h' && src[0] == 'h')
2959                         instr->type = INSTR_ALU_OR_HH;
2960
2961                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2962                 instr->alu.dst.n_bits = fdst->n_bits;
2963                 instr->alu.dst.offset = fdst->offset / 8;
2964                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2965                 instr->alu.src.n_bits = fsrc->n_bits;
2966                 instr->alu.src.offset = fsrc->offset / 8;
2967                 return 0;
2968         }
2969
2970         /* OR_I. */
2971         src_val = strtoull(src, &src, 0);
2972         CHECK(!src[0], EINVAL);
2973
2974         if (dst[0] == 'h')
2975                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2976
2977         instr->type = INSTR_ALU_OR_I;
2978         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2979         instr->alu.dst.n_bits = fdst->n_bits;
2980         instr->alu.dst.offset = fdst->offset / 8;
2981         instr->alu.src_val = src_val;
2982         return 0;
2983 }
2984
2985 static int
2986 instr_alu_xor_translate(struct rte_swx_pipeline *p,
2987                         struct action *action,
2988                         char **tokens,
2989                         int n_tokens,
2990                         struct instruction *instr,
2991                         struct instruction_data *data __rte_unused)
2992 {
2993         char *dst = tokens[1], *src = tokens[2];
2994         struct field *fdst, *fsrc;
2995         uint64_t src_val;
2996         uint32_t dst_struct_id = 0, src_struct_id = 0;
2997
2998         CHECK(n_tokens == 3, EINVAL);
2999
3000         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3001         CHECK(fdst, EINVAL);
3002         CHECK(!fdst->var_size, EINVAL);
3003
3004         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3006         if (fsrc) {
3007                 CHECK(!fsrc->var_size, EINVAL);
3008
3009                 instr->type = INSTR_ALU_XOR;
3010                 if (dst[0] != 'h' && src[0] == 'h')
3011                         instr->type = INSTR_ALU_XOR_MH;
3012                 if (dst[0] == 'h' && src[0] != 'h')
3013                         instr->type = INSTR_ALU_XOR_HM;
3014                 if (dst[0] == 'h' && src[0] == 'h')
3015                         instr->type = INSTR_ALU_XOR_HH;
3016
3017                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3018                 instr->alu.dst.n_bits = fdst->n_bits;
3019                 instr->alu.dst.offset = fdst->offset / 8;
3020                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3021                 instr->alu.src.n_bits = fsrc->n_bits;
3022                 instr->alu.src.offset = fsrc->offset / 8;
3023                 return 0;
3024         }
3025
3026         /* XOR_I. */
3027         src_val = strtoull(src, &src, 0);
3028         CHECK(!src[0], EINVAL);
3029
3030         if (dst[0] == 'h')
3031                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3032
3033         instr->type = INSTR_ALU_XOR_I;
3034         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3035         instr->alu.dst.n_bits = fdst->n_bits;
3036         instr->alu.dst.offset = fdst->offset / 8;
3037         instr->alu.src_val = src_val;
3038         return 0;
3039 }
3040
3041 static inline void
3042 instr_alu_add_exec(struct rte_swx_pipeline *p)
3043 {
3044         struct thread *t = &p->threads[p->thread_id];
3045         struct instruction *ip = t->ip;
3046
3047         /* Structs */
3048         __instr_alu_add_exec(p, t, ip);
3049
3050         /* Thread. */
3051         thread_ip_inc(p);
3052 }
3053
3054 static inline void
3055 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3056 {
3057         struct thread *t = &p->threads[p->thread_id];
3058         struct instruction *ip = t->ip;
3059
3060         /* Structs. */
3061         __instr_alu_add_mh_exec(p, t, ip);
3062
3063         /* Thread. */
3064         thread_ip_inc(p);
3065 }
3066
3067 static inline void
3068 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3069 {
3070         struct thread *t = &p->threads[p->thread_id];
3071         struct instruction *ip = t->ip;
3072
3073         /* Structs. */
3074         __instr_alu_add_hm_exec(p, t, ip);
3075
3076         /* Thread. */
3077         thread_ip_inc(p);
3078 }
3079
3080 static inline void
3081 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3082 {
3083         struct thread *t = &p->threads[p->thread_id];
3084         struct instruction *ip = t->ip;
3085
3086         /* Structs. */
3087         __instr_alu_add_hh_exec(p, t, ip);
3088
3089         /* Thread. */
3090         thread_ip_inc(p);
3091 }
3092
3093 static inline void
3094 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3095 {
3096         struct thread *t = &p->threads[p->thread_id];
3097         struct instruction *ip = t->ip;
3098
3099         /* Structs. */
3100         __instr_alu_add_mi_exec(p, t, ip);
3101
3102         /* Thread. */
3103         thread_ip_inc(p);
3104 }
3105
3106 static inline void
3107 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3108 {
3109         struct thread *t = &p->threads[p->thread_id];
3110         struct instruction *ip = t->ip;
3111
3112         /* Structs. */
3113         __instr_alu_add_hi_exec(p, t, ip);
3114
3115         /* Thread. */
3116         thread_ip_inc(p);
3117 }
3118
3119 static inline void
3120 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3121 {
3122         struct thread *t = &p->threads[p->thread_id];
3123         struct instruction *ip = t->ip;
3124
3125         /* Structs. */
3126         __instr_alu_sub_exec(p, t, ip);
3127
3128         /* Thread. */
3129         thread_ip_inc(p);
3130 }
3131
3132 static inline void
3133 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3134 {
3135         struct thread *t = &p->threads[p->thread_id];
3136         struct instruction *ip = t->ip;
3137
3138         /* Structs. */
3139         __instr_alu_sub_mh_exec(p, t, ip);
3140
3141         /* Thread. */
3142         thread_ip_inc(p);
3143 }
3144
3145 static inline void
3146 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3147 {
3148         struct thread *t = &p->threads[p->thread_id];
3149         struct instruction *ip = t->ip;
3150
3151         /* Structs. */
3152         __instr_alu_sub_hm_exec(p, t, ip);
3153
3154         /* Thread. */
3155         thread_ip_inc(p);
3156 }
3157
3158 static inline void
3159 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3160 {
3161         struct thread *t = &p->threads[p->thread_id];
3162         struct instruction *ip = t->ip;
3163
3164         /* Structs. */
3165         __instr_alu_sub_hh_exec(p, t, ip);
3166
3167         /* Thread. */
3168         thread_ip_inc(p);
3169 }
3170
3171 static inline void
3172 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3173 {
3174         struct thread *t = &p->threads[p->thread_id];
3175         struct instruction *ip = t->ip;
3176
3177         /* Structs. */
3178         __instr_alu_sub_mi_exec(p, t, ip);
3179
3180         /* Thread. */
3181         thread_ip_inc(p);
3182 }
3183
3184 static inline void
3185 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3186 {
3187         struct thread *t = &p->threads[p->thread_id];
3188         struct instruction *ip = t->ip;
3189
3190         /* Structs. */
3191         __instr_alu_sub_hi_exec(p, t, ip);
3192
3193         /* Thread. */
3194         thread_ip_inc(p);
3195 }
3196
3197 static inline void
3198 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3199 {
3200         struct thread *t = &p->threads[p->thread_id];
3201         struct instruction *ip = t->ip;
3202
3203         /* Structs. */
3204         __instr_alu_shl_exec(p, t, ip);
3205
3206         /* Thread. */
3207         thread_ip_inc(p);
3208 }
3209
3210 static inline void
3211 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3212 {
3213         struct thread *t = &p->threads[p->thread_id];
3214         struct instruction *ip = t->ip;
3215
3216         /* Structs. */
3217         __instr_alu_shl_mh_exec(p, t, ip);
3218
3219         /* Thread. */
3220         thread_ip_inc(p);
3221 }
3222
3223 static inline void
3224 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3225 {
3226         struct thread *t = &p->threads[p->thread_id];
3227         struct instruction *ip = t->ip;
3228
3229         /* Structs. */
3230         __instr_alu_shl_hm_exec(p, t, ip);
3231
3232         /* Thread. */
3233         thread_ip_inc(p);
3234 }
3235
3236 static inline void
3237 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3238 {
3239         struct thread *t = &p->threads[p->thread_id];
3240         struct instruction *ip = t->ip;
3241
3242         /* Structs. */
3243         __instr_alu_shl_hh_exec(p, t, ip);
3244
3245         /* Thread. */
3246         thread_ip_inc(p);
3247 }
3248
3249 static inline void
3250 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3251 {
3252         struct thread *t = &p->threads[p->thread_id];
3253         struct instruction *ip = t->ip;
3254
3255         /* Structs. */
3256         __instr_alu_shl_mi_exec(p, t, ip);
3257
3258         /* Thread. */
3259         thread_ip_inc(p);
3260 }
3261
3262 static inline void
3263 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3264 {
3265         struct thread *t = &p->threads[p->thread_id];
3266         struct instruction *ip = t->ip;
3267
3268         /* Structs. */
3269         __instr_alu_shl_hi_exec(p, t, ip);
3270
3271         /* Thread. */
3272         thread_ip_inc(p);
3273 }
3274
3275 static inline void
3276 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3277 {
3278         struct thread *t = &p->threads[p->thread_id];
3279         struct instruction *ip = t->ip;
3280
3281         /* Structs. */
3282         __instr_alu_shr_exec(p, t, ip);
3283
3284         /* Thread. */
3285         thread_ip_inc(p);
3286 }
3287
3288 static inline void
3289 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3290 {
3291         struct thread *t = &p->threads[p->thread_id];
3292         struct instruction *ip = t->ip;
3293
3294         /* Structs. */
3295         __instr_alu_shr_mh_exec(p, t, ip);
3296
3297         /* Thread. */
3298         thread_ip_inc(p);
3299 }
3300
3301 static inline void
3302 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3303 {
3304         struct thread *t = &p->threads[p->thread_id];
3305         struct instruction *ip = t->ip;
3306
3307         /* Structs. */
3308         __instr_alu_shr_hm_exec(p, t, ip);
3309
3310         /* Thread. */
3311         thread_ip_inc(p);
3312 }
3313
3314 static inline void
3315 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3316 {
3317         struct thread *t = &p->threads[p->thread_id];
3318         struct instruction *ip = t->ip;
3319
3320         /* Structs. */
3321         __instr_alu_shr_hh_exec(p, t, ip);
3322
3323         /* Thread. */
3324         thread_ip_inc(p);
3325 }
3326
3327 static inline void
3328 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3329 {
3330         struct thread *t = &p->threads[p->thread_id];
3331         struct instruction *ip = t->ip;
3332
3333         /* Structs. */
3334         __instr_alu_shr_mi_exec(p, t, ip);
3335
3336         /* Thread. */
3337         thread_ip_inc(p);
3338 }
3339
3340 static inline void
3341 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3342 {
3343         struct thread *t = &p->threads[p->thread_id];
3344         struct instruction *ip = t->ip;
3345
3346         /* Structs. */
3347         __instr_alu_shr_hi_exec(p, t, ip);
3348
3349         /* Thread. */
3350         thread_ip_inc(p);
3351 }
3352
3353 static inline void
3354 instr_alu_and_exec(struct rte_swx_pipeline *p)
3355 {
3356         struct thread *t = &p->threads[p->thread_id];
3357         struct instruction *ip = t->ip;
3358
3359         /* Structs. */
3360         __instr_alu_and_exec(p, t, ip);
3361
3362         /* Thread. */
3363         thread_ip_inc(p);
3364 }
3365
3366 static inline void
3367 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3368 {
3369         struct thread *t = &p->threads[p->thread_id];
3370         struct instruction *ip = t->ip;
3371
3372         /* Structs. */
3373         __instr_alu_and_mh_exec(p, t, ip);
3374
3375         /* Thread. */
3376         thread_ip_inc(p);
3377 }
3378
3379 static inline void
3380 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3381 {
3382         struct thread *t = &p->threads[p->thread_id];
3383         struct instruction *ip = t->ip;
3384
3385         /* Structs. */
3386         __instr_alu_and_hm_exec(p, t, ip);
3387
3388         /* Thread. */
3389         thread_ip_inc(p);
3390 }
3391
3392 static inline void
3393 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3394 {
3395         struct thread *t = &p->threads[p->thread_id];
3396         struct instruction *ip = t->ip;
3397
3398         /* Structs. */
3399         __instr_alu_and_hh_exec(p, t, ip);
3400
3401         /* Thread. */
3402         thread_ip_inc(p);
3403 }
3404
3405 static inline void
3406 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3407 {
3408         struct thread *t = &p->threads[p->thread_id];
3409         struct instruction *ip = t->ip;
3410
3411         /* Structs. */
3412         __instr_alu_and_i_exec(p, t, ip);
3413
3414         /* Thread. */
3415         thread_ip_inc(p);
3416 }
3417
3418 static inline void
3419 instr_alu_or_exec(struct rte_swx_pipeline *p)
3420 {
3421         struct thread *t = &p->threads[p->thread_id];
3422         struct instruction *ip = t->ip;
3423
3424         /* Structs. */
3425         __instr_alu_or_exec(p, t, ip);
3426
3427         /* Thread. */
3428         thread_ip_inc(p);
3429 }
3430
3431 static inline void
3432 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3433 {
3434         struct thread *t = &p->threads[p->thread_id];
3435         struct instruction *ip = t->ip;
3436
3437         /* Structs. */
3438         __instr_alu_or_mh_exec(p, t, ip);
3439
3440         /* Thread. */
3441         thread_ip_inc(p);
3442 }
3443
3444 static inline void
3445 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3446 {
3447         struct thread *t = &p->threads[p->thread_id];
3448         struct instruction *ip = t->ip;
3449
3450         /* Structs. */
3451         __instr_alu_or_hm_exec(p, t, ip);
3452
3453         /* Thread. */
3454         thread_ip_inc(p);
3455 }
3456
3457 static inline void
3458 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3459 {
3460         struct thread *t = &p->threads[p->thread_id];
3461         struct instruction *ip = t->ip;
3462
3463         /* Structs. */
3464         __instr_alu_or_hh_exec(p, t, ip);
3465
3466         /* Thread. */
3467         thread_ip_inc(p);
3468 }
3469
3470 static inline void
3471 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3472 {
3473         struct thread *t = &p->threads[p->thread_id];
3474         struct instruction *ip = t->ip;
3475
3476         /* Structs. */
3477         __instr_alu_or_i_exec(p, t, ip);
3478
3479         /* Thread. */
3480         thread_ip_inc(p);
3481 }
3482
3483 static inline void
3484 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3485 {
3486         struct thread *t = &p->threads[p->thread_id];
3487         struct instruction *ip = t->ip;
3488
3489         /* Structs. */
3490         __instr_alu_xor_exec(p, t, ip);
3491
3492         /* Thread. */
3493         thread_ip_inc(p);
3494 }
3495
3496 static inline void
3497 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3498 {
3499         struct thread *t = &p->threads[p->thread_id];
3500         struct instruction *ip = t->ip;
3501
3502         /* Structs. */
3503         __instr_alu_xor_mh_exec(p, t, ip);
3504
3505         /* Thread. */
3506         thread_ip_inc(p);
3507 }
3508
3509 static inline void
3510 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3511 {
3512         struct thread *t = &p->threads[p->thread_id];
3513         struct instruction *ip = t->ip;
3514
3515         /* Structs. */
3516         __instr_alu_xor_hm_exec(p, t, ip);
3517
3518         /* Thread. */
3519         thread_ip_inc(p);
3520 }
3521
3522 static inline void
3523 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3524 {
3525         struct thread *t = &p->threads[p->thread_id];
3526         struct instruction *ip = t->ip;
3527
3528         /* Structs. */
3529         __instr_alu_xor_hh_exec(p, t, ip);
3530
3531         /* Thread. */
3532         thread_ip_inc(p);
3533 }
3534
3535 static inline void
3536 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3537 {
3538         struct thread *t = &p->threads[p->thread_id];
3539         struct instruction *ip = t->ip;
3540
3541         /* Structs. */
3542         __instr_alu_xor_i_exec(p, t, ip);
3543
3544         /* Thread. */
3545         thread_ip_inc(p);
3546 }
3547
3548 static inline void
3549 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3550 {
3551         struct thread *t = &p->threads[p->thread_id];
3552         struct instruction *ip = t->ip;
3553
3554         /* Structs. */
3555         __instr_alu_ckadd_field_exec(p, t, ip);
3556
3557         /* Thread. */
3558         thread_ip_inc(p);
3559 }
3560
3561 static inline void
3562 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3563 {
3564         struct thread *t = &p->threads[p->thread_id];
3565         struct instruction *ip = t->ip;
3566
3567         /* Structs. */
3568         __instr_alu_cksub_field_exec(p, t, ip);
3569
3570         /* Thread. */
3571         thread_ip_inc(p);
3572 }
3573
3574 static inline void
3575 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3576 {
3577         struct thread *t = &p->threads[p->thread_id];
3578         struct instruction *ip = t->ip;
3579
3580         /* Structs. */
3581         __instr_alu_ckadd_struct20_exec(p, t, ip);
3582
3583         /* Thread. */
3584         thread_ip_inc(p);
3585 }
3586
3587 static inline void
3588 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3589 {
3590         struct thread *t = &p->threads[p->thread_id];
3591         struct instruction *ip = t->ip;
3592
3593         /* Structs. */
3594         __instr_alu_ckadd_struct_exec(p, t, ip);
3595
3596         /* Thread. */
3597         thread_ip_inc(p);
3598 }
3599
3600 /*
3601  * Register array.
3602  */
3603 static struct regarray *
3604 regarray_find(struct rte_swx_pipeline *p, const char *name);
3605
3606 static int
3607 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3608                       struct action *action,
3609                       char **tokens,
3610                       int n_tokens,
3611                       struct instruction *instr,
3612                       struct instruction_data *data __rte_unused)
3613 {
3614         char *regarray = tokens[1], *idx = tokens[2];
3615         struct regarray *r;
3616         struct field *fidx;
3617         uint32_t idx_struct_id, idx_val;
3618
3619         CHECK(n_tokens == 3, EINVAL);
3620
3621         r = regarray_find(p, regarray);
3622         CHECK(r, EINVAL);
3623
3624         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3625         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3626         if (fidx) {
3627                 CHECK(!fidx->var_size, EINVAL);
3628
3629                 instr->type = INSTR_REGPREFETCH_RM;
3630                 if (idx[0] == 'h')
3631                         instr->type = INSTR_REGPREFETCH_RH;
3632
3633                 instr->regarray.regarray_id = r->id;
3634                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3635                 instr->regarray.idx.n_bits = fidx->n_bits;
3636                 instr->regarray.idx.offset = fidx->offset / 8;
3637                 instr->regarray.dstsrc_val = 0; /* Unused. */
3638                 return 0;
3639         }
3640
3641         /* REGPREFETCH_RI. */
3642         idx_val = strtoul(idx, &idx, 0);
3643         CHECK(!idx[0], EINVAL);
3644
3645         instr->type = INSTR_REGPREFETCH_RI;
3646         instr->regarray.regarray_id = r->id;
3647         instr->regarray.idx_val = idx_val;
3648         instr->regarray.dstsrc_val = 0; /* Unused. */
3649         return 0;
3650 }
3651
3652 static int
3653 instr_regrd_translate(struct rte_swx_pipeline *p,
3654                       struct action *action,
3655                       char **tokens,
3656                       int n_tokens,
3657                       struct instruction *instr,
3658                       struct instruction_data *data __rte_unused)
3659 {
3660         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3661         struct regarray *r;
3662         struct field *fdst, *fidx;
3663         uint32_t dst_struct_id, idx_struct_id, idx_val;
3664
3665         CHECK(n_tokens == 4, EINVAL);
3666
3667         r = regarray_find(p, regarray);
3668         CHECK(r, EINVAL);
3669
3670         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3671         CHECK(fdst, EINVAL);
3672         CHECK(!fdst->var_size, EINVAL);
3673
3674         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3675         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3676         if (fidx) {
3677                 CHECK(!fidx->var_size, EINVAL);
3678
3679                 instr->type = INSTR_REGRD_MRM;
3680                 if (dst[0] == 'h' && idx[0] != 'h')
3681                         instr->type = INSTR_REGRD_HRM;
3682                 if (dst[0] != 'h' && idx[0] == 'h')
3683                         instr->type = INSTR_REGRD_MRH;
3684                 if (dst[0] == 'h' && idx[0] == 'h')
3685                         instr->type = INSTR_REGRD_HRH;
3686
3687                 instr->regarray.regarray_id = r->id;
3688                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3689                 instr->regarray.idx.n_bits = fidx->n_bits;
3690                 instr->regarray.idx.offset = fidx->offset / 8;
3691                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3692                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3693                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3694                 return 0;
3695         }
3696
3697         /* REGRD_MRI, REGRD_HRI. */
3698         idx_val = strtoul(idx, &idx, 0);
3699         CHECK(!idx[0], EINVAL);
3700
3701         instr->type = INSTR_REGRD_MRI;
3702         if (dst[0] == 'h')
3703                 instr->type = INSTR_REGRD_HRI;
3704
3705         instr->regarray.regarray_id = r->id;
3706         instr->regarray.idx_val = idx_val;
3707         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3708         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3709         instr->regarray.dstsrc.offset = fdst->offset / 8;
3710         return 0;
3711 }
3712
3713 static int
3714 instr_regwr_translate(struct rte_swx_pipeline *p,
3715                       struct action *action,
3716                       char **tokens,
3717                       int n_tokens,
3718                       struct instruction *instr,
3719                       struct instruction_data *data __rte_unused)
3720 {
3721         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3722         struct regarray *r;
3723         struct field *fidx, *fsrc;
3724         uint64_t src_val;
3725         uint32_t idx_struct_id, idx_val, src_struct_id;
3726
3727         CHECK(n_tokens == 4, EINVAL);
3728
3729         r = regarray_find(p, regarray);
3730         CHECK(r, EINVAL);
3731
3732         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3733         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3734         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3735         if (fidx && fsrc) {
3736                 CHECK(!fidx->var_size, EINVAL);
3737                 CHECK(!fsrc->var_size, EINVAL);
3738
3739                 instr->type = INSTR_REGWR_RMM;
3740                 if (idx[0] == 'h' && src[0] != 'h')
3741                         instr->type = INSTR_REGWR_RHM;
3742                 if (idx[0] != 'h' && src[0] == 'h')
3743                         instr->type = INSTR_REGWR_RMH;
3744                 if (idx[0] == 'h' && src[0] == 'h')
3745                         instr->type = INSTR_REGWR_RHH;
3746
3747                 instr->regarray.regarray_id = r->id;
3748                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3749                 instr->regarray.idx.n_bits = fidx->n_bits;
3750                 instr->regarray.idx.offset = fidx->offset / 8;
3751                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3752                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3753                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3754                 return 0;
3755         }
3756
3757         /* REGWR_RHI, REGWR_RMI. */
3758         if (fidx && !fsrc) {
3759                 CHECK(!fidx->var_size, EINVAL);
3760
3761                 src_val = strtoull(src, &src, 0);
3762                 CHECK(!src[0], EINVAL);
3763
3764                 instr->type = INSTR_REGWR_RMI;
3765                 if (idx[0] == 'h')
3766                         instr->type = INSTR_REGWR_RHI;
3767
3768                 instr->regarray.regarray_id = r->id;
3769                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3770                 instr->regarray.idx.n_bits = fidx->n_bits;
3771                 instr->regarray.idx.offset = fidx->offset / 8;
3772                 instr->regarray.dstsrc_val = src_val;
3773                 return 0;
3774         }
3775
3776         /* REGWR_RIH, REGWR_RIM. */
3777         if (!fidx && fsrc) {
3778                 idx_val = strtoul(idx, &idx, 0);
3779                 CHECK(!idx[0], EINVAL);
3780
3781                 CHECK(!fsrc->var_size, EINVAL);
3782
3783                 instr->type = INSTR_REGWR_RIM;
3784                 if (src[0] == 'h')
3785                         instr->type = INSTR_REGWR_RIH;
3786
3787                 instr->regarray.regarray_id = r->id;
3788                 instr->regarray.idx_val = idx_val;
3789                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3790                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3791                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3792                 return 0;
3793         }
3794
3795         /* REGWR_RII. */
3796         src_val = strtoull(src, &src, 0);
3797         CHECK(!src[0], EINVAL);
3798
3799         idx_val = strtoul(idx, &idx, 0);
3800         CHECK(!idx[0], EINVAL);
3801
3802         instr->type = INSTR_REGWR_RII;
3803         instr->regarray.idx_val = idx_val;
3804         instr->regarray.dstsrc_val = src_val;
3805
3806         return 0;
3807 }
3808
3809 static int
3810 instr_regadd_translate(struct rte_swx_pipeline *p,
3811                        struct action *action,
3812                        char **tokens,
3813                        int n_tokens,
3814                        struct instruction *instr,
3815                        struct instruction_data *data __rte_unused)
3816 {
3817         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3818         struct regarray *r;
3819         struct field *fidx, *fsrc;
3820         uint64_t src_val;
3821         uint32_t idx_struct_id, idx_val, src_struct_id;
3822
3823         CHECK(n_tokens == 4, EINVAL);
3824
3825         r = regarray_find(p, regarray);
3826         CHECK(r, EINVAL);
3827
3828         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3829         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3830         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3831         if (fidx && fsrc) {
3832                 CHECK(!fidx->var_size, EINVAL);
3833                 CHECK(!fsrc->var_size, EINVAL);
3834
3835                 instr->type = INSTR_REGADD_RMM;
3836                 if (idx[0] == 'h' && src[0] != 'h')
3837                         instr->type = INSTR_REGADD_RHM;
3838                 if (idx[0] != 'h' && src[0] == 'h')
3839                         instr->type = INSTR_REGADD_RMH;
3840                 if (idx[0] == 'h' && src[0] == 'h')
3841                         instr->type = INSTR_REGADD_RHH;
3842
3843                 instr->regarray.regarray_id = r->id;
3844                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3845                 instr->regarray.idx.n_bits = fidx->n_bits;
3846                 instr->regarray.idx.offset = fidx->offset / 8;
3847                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3848                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3849                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3850                 return 0;
3851         }
3852
3853         /* REGADD_RHI, REGADD_RMI. */
3854         if (fidx && !fsrc) {
3855                 CHECK(!fidx->var_size, EINVAL);
3856
3857                 src_val = strtoull(src, &src, 0);
3858                 CHECK(!src[0], EINVAL);
3859
3860                 instr->type = INSTR_REGADD_RMI;
3861                 if (idx[0] == 'h')
3862                         instr->type = INSTR_REGADD_RHI;
3863
3864                 instr->regarray.regarray_id = r->id;
3865                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3866                 instr->regarray.idx.n_bits = fidx->n_bits;
3867                 instr->regarray.idx.offset = fidx->offset / 8;
3868                 instr->regarray.dstsrc_val = src_val;
3869                 return 0;
3870         }
3871
3872         /* REGADD_RIH, REGADD_RIM. */
3873         if (!fidx && fsrc) {
3874                 idx_val = strtoul(idx, &idx, 0);
3875                 CHECK(!idx[0], EINVAL);
3876
3877                 CHECK(!fsrc->var_size, EINVAL);
3878
3879                 instr->type = INSTR_REGADD_RIM;
3880                 if (src[0] == 'h')
3881                         instr->type = INSTR_REGADD_RIH;
3882
3883                 instr->regarray.regarray_id = r->id;
3884                 instr->regarray.idx_val = idx_val;
3885                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3886                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3887                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3888                 return 0;
3889         }
3890
3891         /* REGADD_RII. */
3892         src_val = strtoull(src, &src, 0);
3893         CHECK(!src[0], EINVAL);
3894
3895         idx_val = strtoul(idx, &idx, 0);
3896         CHECK(!idx[0], EINVAL);
3897
3898         instr->type = INSTR_REGADD_RII;
3899         instr->regarray.idx_val = idx_val;
3900         instr->regarray.dstsrc_val = src_val;
3901         return 0;
3902 }
3903
3904 static inline void
3905 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
3906 {
3907         struct thread *t = &p->threads[p->thread_id];
3908         struct instruction *ip = t->ip;
3909
3910         /* Structs. */
3911         __instr_regprefetch_rh_exec(p, t, ip);
3912
3913         /* Thread. */
3914         thread_ip_inc(p);
3915 }
3916
3917 static inline void
3918 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
3919 {
3920         struct thread *t = &p->threads[p->thread_id];
3921         struct instruction *ip = t->ip;
3922
3923         /* Structs. */
3924         __instr_regprefetch_rm_exec(p, t, ip);
3925
3926         /* Thread. */
3927         thread_ip_inc(p);
3928 }
3929
3930 static inline void
3931 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
3932 {
3933         struct thread *t = &p->threads[p->thread_id];
3934         struct instruction *ip = t->ip;
3935
3936         /* Structs. */
3937         __instr_regprefetch_ri_exec(p, t, ip);
3938
3939         /* Thread. */
3940         thread_ip_inc(p);
3941 }
3942
3943 static inline void
3944 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
3945 {
3946         struct thread *t = &p->threads[p->thread_id];
3947         struct instruction *ip = t->ip;
3948
3949         /* Structs. */
3950         __instr_regrd_hrh_exec(p, t, ip);
3951
3952         /* Thread. */
3953         thread_ip_inc(p);
3954 }
3955
3956 static inline void
3957 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
3958 {
3959         struct thread *t = &p->threads[p->thread_id];
3960         struct instruction *ip = t->ip;
3961
3962         /* Structs. */
3963         __instr_regrd_hrm_exec(p, t, ip);
3964
3965         /* Thread. */
3966         thread_ip_inc(p);
3967 }
3968
3969 static inline void
3970 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
3971 {
3972         struct thread *t = &p->threads[p->thread_id];
3973         struct instruction *ip = t->ip;
3974
3975         /* Structs. */
3976         __instr_regrd_mrh_exec(p, t, ip);
3977
3978         /* Thread. */
3979         thread_ip_inc(p);
3980 }
3981
3982 static inline void
3983 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
3984 {
3985         struct thread *t = &p->threads[p->thread_id];
3986         struct instruction *ip = t->ip;
3987
3988         /* Structs. */
3989         __instr_regrd_mrm_exec(p, t, ip);
3990
3991         /* Thread. */
3992         thread_ip_inc(p);
3993 }
3994
3995 static inline void
3996 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
3997 {
3998         struct thread *t = &p->threads[p->thread_id];
3999         struct instruction *ip = t->ip;
4000
4001         /* Structs. */
4002         __instr_regrd_hri_exec(p, t, ip);
4003
4004         /* Thread. */
4005         thread_ip_inc(p);
4006 }
4007
4008 static inline void
4009 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4010 {
4011         struct thread *t = &p->threads[p->thread_id];
4012         struct instruction *ip = t->ip;
4013
4014         /* Structs. */
4015         __instr_regrd_mri_exec(p, t, ip);
4016
4017         /* Thread. */
4018         thread_ip_inc(p);
4019 }
4020
4021 static inline void
4022 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4023 {
4024         struct thread *t = &p->threads[p->thread_id];
4025         struct instruction *ip = t->ip;
4026
4027         /* Structs. */
4028         __instr_regwr_rhh_exec(p, t, ip);
4029
4030         /* Thread. */
4031         thread_ip_inc(p);
4032 }
4033
4034 static inline void
4035 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4036 {
4037         struct thread *t = &p->threads[p->thread_id];
4038         struct instruction *ip = t->ip;
4039
4040         /* Structs. */
4041         __instr_regwr_rhm_exec(p, t, ip);
4042
4043         /* Thread. */
4044         thread_ip_inc(p);
4045 }
4046
4047 static inline void
4048 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4049 {
4050         struct thread *t = &p->threads[p->thread_id];
4051         struct instruction *ip = t->ip;
4052
4053         /* Structs. */
4054         __instr_regwr_rmh_exec(p, t, ip);
4055
4056         /* Thread. */
4057         thread_ip_inc(p);
4058 }
4059
4060 static inline void
4061 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4062 {
4063         struct thread *t = &p->threads[p->thread_id];
4064         struct instruction *ip = t->ip;
4065
4066         /* Structs. */
4067         __instr_regwr_rmm_exec(p, t, ip);
4068
4069         /* Thread. */
4070         thread_ip_inc(p);
4071 }
4072
4073 static inline void
4074 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4075 {
4076         struct thread *t = &p->threads[p->thread_id];
4077         struct instruction *ip = t->ip;
4078
4079         /* Structs. */
4080         __instr_regwr_rhi_exec(p, t, ip);
4081
4082         /* Thread. */
4083         thread_ip_inc(p);
4084 }
4085
4086 static inline void
4087 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4088 {
4089         struct thread *t = &p->threads[p->thread_id];
4090         struct instruction *ip = t->ip;
4091
4092         /* Structs. */
4093         __instr_regwr_rmi_exec(p, t, ip);
4094
4095         /* Thread. */
4096         thread_ip_inc(p);
4097 }
4098
4099 static inline void
4100 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4101 {
4102         struct thread *t = &p->threads[p->thread_id];
4103         struct instruction *ip = t->ip;
4104
4105         /* Structs. */
4106         __instr_regwr_rih_exec(p, t, ip);
4107
4108         /* Thread. */
4109         thread_ip_inc(p);
4110 }
4111
4112 static inline void
4113 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4114 {
4115         struct thread *t = &p->threads[p->thread_id];
4116         struct instruction *ip = t->ip;
4117
4118         /* Structs. */
4119         __instr_regwr_rim_exec(p, t, ip);
4120
4121         /* Thread. */
4122         thread_ip_inc(p);
4123 }
4124
4125 static inline void
4126 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4127 {
4128         struct thread *t = &p->threads[p->thread_id];
4129         struct instruction *ip = t->ip;
4130
4131         /* Structs. */
4132         __instr_regwr_rii_exec(p, t, ip);
4133
4134         /* Thread. */
4135         thread_ip_inc(p);
4136 }
4137
4138 static inline void
4139 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4140 {
4141         struct thread *t = &p->threads[p->thread_id];
4142         struct instruction *ip = t->ip;
4143
4144         /* Structs. */
4145         __instr_regadd_rhh_exec(p, t, ip);
4146
4147         /* Thread. */
4148         thread_ip_inc(p);
4149 }
4150
4151 static inline void
4152 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4153 {
4154         struct thread *t = &p->threads[p->thread_id];
4155         struct instruction *ip = t->ip;
4156
4157         /* Structs. */
4158         __instr_regadd_rhm_exec(p, t, ip);
4159
4160         /* Thread. */
4161         thread_ip_inc(p);
4162 }
4163
4164 static inline void
4165 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4166 {
4167         struct thread *t = &p->threads[p->thread_id];
4168         struct instruction *ip = t->ip;
4169
4170         /* Structs. */
4171         __instr_regadd_rmh_exec(p, t, ip);
4172
4173         /* Thread. */
4174         thread_ip_inc(p);
4175 }
4176
4177 static inline void
4178 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4179 {
4180         struct thread *t = &p->threads[p->thread_id];
4181         struct instruction *ip = t->ip;
4182
4183         /* Structs. */
4184         __instr_regadd_rmm_exec(p, t, ip);
4185
4186         /* Thread. */
4187         thread_ip_inc(p);
4188 }
4189
4190 static inline void
4191 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4192 {
4193         struct thread *t = &p->threads[p->thread_id];
4194         struct instruction *ip = t->ip;
4195
4196         /* Structs. */
4197         __instr_regadd_rhi_exec(p, t, ip);
4198
4199         /* Thread. */
4200         thread_ip_inc(p);
4201 }
4202
4203 static inline void
4204 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4205 {
4206         struct thread *t = &p->threads[p->thread_id];
4207         struct instruction *ip = t->ip;
4208
4209         /* Structs. */
4210         __instr_regadd_rmi_exec(p, t, ip);
4211
4212         /* Thread. */
4213         thread_ip_inc(p);
4214 }
4215
4216 static inline void
4217 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4218 {
4219         struct thread *t = &p->threads[p->thread_id];
4220         struct instruction *ip = t->ip;
4221
4222         /* Structs. */
4223         __instr_regadd_rih_exec(p, t, ip);
4224
4225         /* Thread. */
4226         thread_ip_inc(p);
4227 }
4228
4229 static inline void
4230 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4231 {
4232         struct thread *t = &p->threads[p->thread_id];
4233         struct instruction *ip = t->ip;
4234
4235         /* Structs. */
4236         __instr_regadd_rim_exec(p, t, ip);
4237
4238         /* Thread. */
4239         thread_ip_inc(p);
4240 }
4241
4242 static inline void
4243 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4244 {
4245         struct thread *t = &p->threads[p->thread_id];
4246         struct instruction *ip = t->ip;
4247
4248         /* Structs. */
4249         __instr_regadd_rii_exec(p, t, ip);
4250
4251         /* Thread. */
4252         thread_ip_inc(p);
4253 }
4254
4255 /*
4256  * metarray.
4257  */
4258 static struct metarray *
4259 metarray_find(struct rte_swx_pipeline *p, const char *name);
4260
4261 static int
4262 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4263                             struct action *action,
4264                             char **tokens,
4265                             int n_tokens,
4266                             struct instruction *instr,
4267                             struct instruction_data *data __rte_unused)
4268 {
4269         char *metarray = tokens[1], *idx = tokens[2];
4270         struct metarray *m;
4271         struct field *fidx;
4272         uint32_t idx_struct_id, idx_val;
4273
4274         CHECK(n_tokens == 3, EINVAL);
4275
4276         m = metarray_find(p, metarray);
4277         CHECK(m, EINVAL);
4278
4279         /* METPREFETCH_H, METPREFETCH_M. */
4280         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4281         if (fidx) {
4282                 CHECK(!fidx->var_size, EINVAL);
4283
4284                 instr->type = INSTR_METPREFETCH_M;
4285                 if (idx[0] == 'h')
4286                         instr->type = INSTR_METPREFETCH_H;
4287
4288                 instr->meter.metarray_id = m->id;
4289                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4290                 instr->meter.idx.n_bits = fidx->n_bits;
4291                 instr->meter.idx.offset = fidx->offset / 8;
4292                 return 0;
4293         }
4294
4295         /* METPREFETCH_I. */
4296         idx_val = strtoul(idx, &idx, 0);
4297         CHECK(!idx[0], EINVAL);
4298
4299         instr->type = INSTR_METPREFETCH_I;
4300         instr->meter.metarray_id = m->id;
4301         instr->meter.idx_val = idx_val;
4302         return 0;
4303 }
4304
4305 static int
4306 instr_meter_translate(struct rte_swx_pipeline *p,
4307                       struct action *action,
4308                       char **tokens,
4309                       int n_tokens,
4310                       struct instruction *instr,
4311                       struct instruction_data *data __rte_unused)
4312 {
4313         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4314         char *color_in = tokens[4], *color_out = tokens[5];
4315         struct metarray *m;
4316         struct field *fidx, *flength, *fcin, *fcout;
4317         uint32_t idx_struct_id, length_struct_id;
4318         uint32_t color_in_struct_id, color_out_struct_id;
4319
4320         CHECK(n_tokens == 6, EINVAL);
4321
4322         m = metarray_find(p, metarray);
4323         CHECK(m, EINVAL);
4324
4325         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4326
4327         flength = struct_field_parse(p, action, length, &length_struct_id);
4328         CHECK(flength, EINVAL);
4329         CHECK(!flength->var_size, EINVAL);
4330
4331         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4332
4333         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4334         CHECK(fcout, EINVAL);
4335         CHECK(!fcout->var_size, EINVAL);
4336
4337         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4338         if (fidx && fcin) {
4339                 CHECK(!fidx->var_size, EINVAL);
4340                 CHECK(!fcin->var_size, EINVAL);
4341
4342                 instr->type = INSTR_METER_MMM;
4343                 if (idx[0] == 'h' && length[0] == 'h')
4344                         instr->type = INSTR_METER_HHM;
4345                 if (idx[0] == 'h' && length[0] != 'h')
4346                         instr->type = INSTR_METER_HMM;
4347                 if (idx[0] != 'h' && length[0] == 'h')
4348                         instr->type = INSTR_METER_MHM;
4349
4350                 instr->meter.metarray_id = m->id;
4351
4352                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4353                 instr->meter.idx.n_bits = fidx->n_bits;
4354                 instr->meter.idx.offset = fidx->offset / 8;
4355
4356                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4357                 instr->meter.length.n_bits = flength->n_bits;
4358                 instr->meter.length.offset = flength->offset / 8;
4359
4360                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4361                 instr->meter.color_in.n_bits = fcin->n_bits;
4362                 instr->meter.color_in.offset = fcin->offset / 8;
4363
4364                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4365                 instr->meter.color_out.n_bits = fcout->n_bits;
4366                 instr->meter.color_out.offset = fcout->offset / 8;
4367
4368                 return 0;
4369         }
4370
4371         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4372         if (fidx && !fcin) {
4373                 uint32_t color_in_val;
4374
4375                 CHECK(!fidx->var_size, EINVAL);
4376
4377                 color_in_val = strtoul(color_in, &color_in, 0);
4378                 CHECK(!color_in[0], EINVAL);
4379
4380                 instr->type = INSTR_METER_MMI;
4381                 if (idx[0] == 'h' && length[0] == 'h')
4382                         instr->type = INSTR_METER_HHI;
4383                 if (idx[0] == 'h' && length[0] != 'h')
4384                         instr->type = INSTR_METER_HMI;
4385                 if (idx[0] != 'h' && length[0] == 'h')
4386                         instr->type = INSTR_METER_MHI;
4387
4388                 instr->meter.metarray_id = m->id;
4389
4390                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4391                 instr->meter.idx.n_bits = fidx->n_bits;
4392                 instr->meter.idx.offset = fidx->offset / 8;
4393
4394                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4395                 instr->meter.length.n_bits = flength->n_bits;
4396                 instr->meter.length.offset = flength->offset / 8;
4397
4398                 instr->meter.color_in_val = color_in_val;
4399
4400                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4401                 instr->meter.color_out.n_bits = fcout->n_bits;
4402                 instr->meter.color_out.offset = fcout->offset / 8;
4403
4404                 return 0;
4405         }
4406
4407         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4408         if (!fidx && fcin) {
4409                 uint32_t idx_val;
4410
4411                 idx_val = strtoul(idx, &idx, 0);
4412                 CHECK(!idx[0], EINVAL);
4413
4414                 CHECK(!fcin->var_size, EINVAL);
4415
4416                 instr->type = INSTR_METER_IMM;
4417                 if (length[0] == 'h')
4418                         instr->type = INSTR_METER_IHM;
4419
4420                 instr->meter.metarray_id = m->id;
4421
4422                 instr->meter.idx_val = idx_val;
4423
4424                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4425                 instr->meter.length.n_bits = flength->n_bits;
4426                 instr->meter.length.offset = flength->offset / 8;
4427
4428                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4429                 instr->meter.color_in.n_bits = fcin->n_bits;
4430                 instr->meter.color_in.offset = fcin->offset / 8;
4431
4432                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4433                 instr->meter.color_out.n_bits = fcout->n_bits;
4434                 instr->meter.color_out.offset = fcout->offset / 8;
4435
4436                 return 0;
4437         }
4438
4439         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4440         if (!fidx && !fcin) {
4441                 uint32_t idx_val, color_in_val;
4442
4443                 idx_val = strtoul(idx, &idx, 0);
4444                 CHECK(!idx[0], EINVAL);
4445
4446                 color_in_val = strtoul(color_in, &color_in, 0);
4447                 CHECK(!color_in[0], EINVAL);
4448
4449                 instr->type = INSTR_METER_IMI;
4450                 if (length[0] == 'h')
4451                         instr->type = INSTR_METER_IHI;
4452
4453                 instr->meter.metarray_id = m->id;
4454
4455                 instr->meter.idx_val = idx_val;
4456
4457                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4458                 instr->meter.length.n_bits = flength->n_bits;
4459                 instr->meter.length.offset = flength->offset / 8;
4460
4461                 instr->meter.color_in_val = color_in_val;
4462
4463                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4464                 instr->meter.color_out.n_bits = fcout->n_bits;
4465                 instr->meter.color_out.offset = fcout->offset / 8;
4466
4467                 return 0;
4468         }
4469
4470         CHECK(0, EINVAL);
4471 }
4472
4473 static inline void
4474 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4475 {
4476         struct thread *t = &p->threads[p->thread_id];
4477         struct instruction *ip = t->ip;
4478
4479         /* Structs. */
4480         __instr_metprefetch_h_exec(p, t, ip);
4481
4482         /* Thread. */
4483         thread_ip_inc(p);
4484 }
4485
4486 static inline void
4487 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4488 {
4489         struct thread *t = &p->threads[p->thread_id];
4490         struct instruction *ip = t->ip;
4491
4492         /* Structs. */
4493         __instr_metprefetch_m_exec(p, t, ip);
4494
4495         /* Thread. */
4496         thread_ip_inc(p);
4497 }
4498
4499 static inline void
4500 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4501 {
4502         struct thread *t = &p->threads[p->thread_id];
4503         struct instruction *ip = t->ip;
4504
4505         /* Structs. */
4506         __instr_metprefetch_i_exec(p, t, ip);
4507
4508         /* Thread. */
4509         thread_ip_inc(p);
4510 }
4511
4512 static inline void
4513 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4514 {
4515         struct thread *t = &p->threads[p->thread_id];
4516         struct instruction *ip = t->ip;
4517
4518         /* Structs. */
4519         __instr_meter_hhm_exec(p, t, ip);
4520
4521         /* Thread. */
4522         thread_ip_inc(p);
4523 }
4524
4525 static inline void
4526 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4527 {
4528         struct thread *t = &p->threads[p->thread_id];
4529         struct instruction *ip = t->ip;
4530
4531         /* Structs. */
4532         __instr_meter_hhi_exec(p, t, ip);
4533
4534         /* Thread. */
4535         thread_ip_inc(p);
4536 }
4537
4538 static inline void
4539 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4540 {
4541         struct thread *t = &p->threads[p->thread_id];
4542         struct instruction *ip = t->ip;
4543
4544         /* Structs. */
4545         __instr_meter_hmm_exec(p, t, ip);
4546
4547         /* Thread. */
4548         thread_ip_inc(p);
4549 }
4550
4551 static inline void
4552 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4553 {
4554         struct thread *t = &p->threads[p->thread_id];
4555         struct instruction *ip = t->ip;
4556
4557         /* Structs. */
4558         __instr_meter_hmi_exec(p, t, ip);
4559
4560         /* Thread. */
4561         thread_ip_inc(p);
4562 }
4563
4564 static inline void
4565 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4566 {
4567         struct thread *t = &p->threads[p->thread_id];
4568         struct instruction *ip = t->ip;
4569
4570         /* Structs. */
4571         __instr_meter_mhm_exec(p, t, ip);
4572
4573         /* Thread. */
4574         thread_ip_inc(p);
4575 }
4576
4577 static inline void
4578 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4579 {
4580         struct thread *t = &p->threads[p->thread_id];
4581         struct instruction *ip = t->ip;
4582
4583         /* Structs. */
4584         __instr_meter_mhi_exec(p, t, ip);
4585
4586         /* Thread. */
4587         thread_ip_inc(p);
4588 }
4589
4590 static inline void
4591 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4592 {
4593         struct thread *t = &p->threads[p->thread_id];
4594         struct instruction *ip = t->ip;
4595
4596         /* Structs. */
4597         __instr_meter_mmm_exec(p, t, ip);
4598
4599         /* Thread. */
4600         thread_ip_inc(p);
4601 }
4602
4603 static inline void
4604 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4605 {
4606         struct thread *t = &p->threads[p->thread_id];
4607         struct instruction *ip = t->ip;
4608
4609         /* Structs. */
4610         __instr_meter_mmi_exec(p, t, ip);
4611
4612         /* Thread. */
4613         thread_ip_inc(p);
4614 }
4615
4616 static inline void
4617 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4618 {
4619         struct thread *t = &p->threads[p->thread_id];
4620         struct instruction *ip = t->ip;
4621
4622         /* Structs. */
4623         __instr_meter_ihm_exec(p, t, ip);
4624
4625         /* Thread. */
4626         thread_ip_inc(p);
4627 }
4628
4629 static inline void
4630 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4631 {
4632         struct thread *t = &p->threads[p->thread_id];
4633         struct instruction *ip = t->ip;
4634
4635         /* Structs. */
4636         __instr_meter_ihi_exec(p, t, ip);
4637
4638         /* Thread. */
4639         thread_ip_inc(p);
4640 }
4641
4642 static inline void
4643 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4644 {
4645         struct thread *t = &p->threads[p->thread_id];
4646         struct instruction *ip = t->ip;
4647
4648         /* Structs. */
4649         __instr_meter_imm_exec(p, t, ip);
4650
4651         /* Thread. */
4652         thread_ip_inc(p);
4653 }
4654
4655 static inline void
4656 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4657 {
4658         struct thread *t = &p->threads[p->thread_id];
4659         struct instruction *ip = t->ip;
4660
4661         /* Structs. */
4662         __instr_meter_imi_exec(p, t, ip);
4663
4664         /* Thread. */
4665         thread_ip_inc(p);
4666 }
4667
4668 /*
4669  * jmp.
4670  */
4671 static int
4672 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4673                     struct action *action __rte_unused,
4674                     char **tokens,
4675                     int n_tokens,
4676                     struct instruction *instr,
4677                     struct instruction_data *data)
4678 {
4679         CHECK(n_tokens == 2, EINVAL);
4680
4681         strcpy(data->jmp_label, tokens[1]);
4682
4683         instr->type = INSTR_JMP;
4684         instr->jmp.ip = NULL; /* Resolved later. */
4685         return 0;
4686 }
4687
4688 static int
4689 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4690                           struct action *action __rte_unused,
4691                           char **tokens,
4692                           int n_tokens,
4693                           struct instruction *instr,
4694                           struct instruction_data *data)
4695 {
4696         struct header *h;
4697
4698         CHECK(n_tokens == 3, EINVAL);
4699
4700         strcpy(data->jmp_label, tokens[1]);
4701
4702         h = header_parse(p, tokens[2]);
4703         CHECK(h, EINVAL);
4704
4705         instr->type = INSTR_JMP_VALID;
4706         instr->jmp.ip = NULL; /* Resolved later. */
4707         instr->jmp.header_id = h->id;
4708         return 0;
4709 }
4710
4711 static int
4712 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4713                             struct action *action __rte_unused,
4714                             char **tokens,
4715                             int n_tokens,
4716                             struct instruction *instr,
4717                             struct instruction_data *data)
4718 {
4719         struct header *h;
4720
4721         CHECK(n_tokens == 3, EINVAL);
4722
4723         strcpy(data->jmp_label, tokens[1]);
4724
4725         h = header_parse(p, tokens[2]);
4726         CHECK(h, EINVAL);
4727
4728         instr->type = INSTR_JMP_INVALID;
4729         instr->jmp.ip = NULL; /* Resolved later. */
4730         instr->jmp.header_id = h->id;
4731         return 0;
4732 }
4733
4734 static int
4735 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4736                         struct action *action,
4737                         char **tokens,
4738                         int n_tokens,
4739                         struct instruction *instr,
4740                         struct instruction_data *data)
4741 {
4742         CHECK(!action, EINVAL);
4743         CHECK(n_tokens == 2, EINVAL);
4744
4745         strcpy(data->jmp_label, tokens[1]);
4746
4747         instr->type = INSTR_JMP_HIT;
4748         instr->jmp.ip = NULL; /* Resolved later. */
4749         return 0;
4750 }
4751
4752 static int
4753 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4754                          struct action *action,
4755                          char **tokens,
4756                          int n_tokens,
4757                          struct instruction *instr,
4758                          struct instruction_data *data)
4759 {
4760         CHECK(!action, EINVAL);
4761         CHECK(n_tokens == 2, EINVAL);
4762
4763         strcpy(data->jmp_label, tokens[1]);
4764
4765         instr->type = INSTR_JMP_MISS;
4766         instr->jmp.ip = NULL; /* Resolved later. */
4767         return 0;
4768 }
4769
4770 static int
4771 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4772                                struct action *action,
4773                                char **tokens,
4774                                int n_tokens,
4775                                struct instruction *instr,
4776                                struct instruction_data *data)
4777 {
4778         struct action *a;
4779
4780         CHECK(!action, EINVAL);
4781         CHECK(n_tokens == 3, EINVAL);
4782
4783         strcpy(data->jmp_label, tokens[1]);
4784
4785         a = action_find(p, tokens[2]);
4786         CHECK(a, EINVAL);
4787
4788         instr->type = INSTR_JMP_ACTION_HIT;
4789         instr->jmp.ip = NULL; /* Resolved later. */
4790         instr->jmp.action_id = a->id;
4791         return 0;
4792 }
4793
4794 static int
4795 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4796                                 struct action *action,
4797                                 char **tokens,
4798                                 int n_tokens,
4799                                 struct instruction *instr,
4800                                 struct instruction_data *data)
4801 {
4802         struct action *a;
4803
4804         CHECK(!action, EINVAL);
4805         CHECK(n_tokens == 3, EINVAL);
4806
4807         strcpy(data->jmp_label, tokens[1]);
4808
4809         a = action_find(p, tokens[2]);
4810         CHECK(a, EINVAL);
4811
4812         instr->type = INSTR_JMP_ACTION_MISS;
4813         instr->jmp.ip = NULL; /* Resolved later. */
4814         instr->jmp.action_id = a->id;
4815         return 0;
4816 }
4817
4818 static int
4819 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4820                        struct action *action,
4821                        char **tokens,
4822                        int n_tokens,
4823                        struct instruction *instr,
4824                        struct instruction_data *data)
4825 {
4826         char *a = tokens[2], *b = tokens[3];
4827         struct field *fa, *fb;
4828         uint64_t b_val;
4829         uint32_t a_struct_id, b_struct_id;
4830
4831         CHECK(n_tokens == 4, EINVAL);
4832
4833         strcpy(data->jmp_label, tokens[1]);
4834
4835         fa = struct_field_parse(p, action, a, &a_struct_id);
4836         CHECK(fa, EINVAL);
4837         CHECK(!fa->var_size, EINVAL);
4838
4839         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
4840         fb = struct_field_parse(p, action, b, &b_struct_id);
4841         if (fb) {
4842                 CHECK(!fb->var_size, EINVAL);
4843
4844                 instr->type = INSTR_JMP_EQ;
4845                 if (a[0] != 'h' && b[0] == 'h')
4846                         instr->type = INSTR_JMP_EQ_MH;
4847                 if (a[0] == 'h' && b[0] != 'h')
4848                         instr->type = INSTR_JMP_EQ_HM;
4849                 if (a[0] == 'h' && b[0] == 'h')
4850                         instr->type = INSTR_JMP_EQ_HH;
4851                 instr->jmp.ip = NULL; /* Resolved later. */
4852
4853                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4854                 instr->jmp.a.n_bits = fa->n_bits;
4855                 instr->jmp.a.offset = fa->offset / 8;
4856                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4857                 instr->jmp.b.n_bits = fb->n_bits;
4858                 instr->jmp.b.offset = fb->offset / 8;
4859                 return 0;
4860         }
4861
4862         /* JMP_EQ_I. */
4863         b_val = strtoull(b, &b, 0);
4864         CHECK(!b[0], EINVAL);
4865
4866         if (a[0] == 'h')
4867                 b_val = hton64(b_val) >> (64 - fa->n_bits);
4868
4869         instr->type = INSTR_JMP_EQ_I;
4870         instr->jmp.ip = NULL; /* Resolved later. */
4871         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4872         instr->jmp.a.n_bits = fa->n_bits;
4873         instr->jmp.a.offset = fa->offset / 8;
4874         instr->jmp.b_val = b_val;
4875         return 0;
4876 }
4877
4878 static int
4879 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
4880                         struct action *action,
4881                         char **tokens,
4882                         int n_tokens,
4883                         struct instruction *instr,
4884                         struct instruction_data *data)
4885 {
4886         char *a = tokens[2], *b = tokens[3];
4887         struct field *fa, *fb;
4888         uint64_t b_val;
4889         uint32_t a_struct_id, b_struct_id;
4890
4891         CHECK(n_tokens == 4, EINVAL);
4892
4893         strcpy(data->jmp_label, tokens[1]);
4894
4895         fa = struct_field_parse(p, action, a, &a_struct_id);
4896         CHECK(fa, EINVAL);
4897         CHECK(!fa->var_size, EINVAL);
4898
4899         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
4900         fb = struct_field_parse(p, action, b, &b_struct_id);
4901         if (fb) {
4902                 CHECK(!fb->var_size, EINVAL);
4903
4904                 instr->type = INSTR_JMP_NEQ;
4905                 if (a[0] != 'h' && b[0] == 'h')
4906                         instr->type = INSTR_JMP_NEQ_MH;
4907                 if (a[0] == 'h' && b[0] != 'h')
4908                         instr->type = INSTR_JMP_NEQ_HM;
4909                 if (a[0] == 'h' && b[0] == 'h')
4910                         instr->type = INSTR_JMP_NEQ_HH;
4911                 instr->jmp.ip = NULL; /* Resolved later. */
4912
4913                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4914                 instr->jmp.a.n_bits = fa->n_bits;
4915                 instr->jmp.a.offset = fa->offset / 8;
4916                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4917                 instr->jmp.b.n_bits = fb->n_bits;
4918                 instr->jmp.b.offset = fb->offset / 8;
4919                 return 0;
4920         }
4921
4922         /* JMP_NEQ_I. */
4923         b_val = strtoull(b, &b, 0);
4924         CHECK(!b[0], EINVAL);
4925
4926         if (a[0] == 'h')
4927                 b_val = hton64(b_val) >> (64 - fa->n_bits);
4928
4929         instr->type = INSTR_JMP_NEQ_I;
4930         instr->jmp.ip = NULL; /* Resolved later. */
4931         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4932         instr->jmp.a.n_bits = fa->n_bits;
4933         instr->jmp.a.offset = fa->offset / 8;
4934         instr->jmp.b_val = b_val;
4935         return 0;
4936 }
4937
4938 static int
4939 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
4940                        struct action *action,
4941                        char **tokens,
4942                        int n_tokens,
4943                        struct instruction *instr,
4944                        struct instruction_data *data)
4945 {
4946         char *a = tokens[2], *b = tokens[3];
4947         struct field *fa, *fb;
4948         uint64_t b_val;
4949         uint32_t a_struct_id, b_struct_id;
4950
4951         CHECK(n_tokens == 4, EINVAL);
4952
4953         strcpy(data->jmp_label, tokens[1]);
4954
4955         fa = struct_field_parse(p, action, a, &a_struct_id);
4956         CHECK(fa, EINVAL);
4957         CHECK(!fa->var_size, EINVAL);
4958
4959         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
4960         fb = struct_field_parse(p, action, b, &b_struct_id);
4961         if (fb) {
4962                 CHECK(!fb->var_size, EINVAL);
4963
4964                 instr->type = INSTR_JMP_LT;
4965                 if (a[0] == 'h' && b[0] != 'h')
4966                         instr->type = INSTR_JMP_LT_HM;
4967                 if (a[0] != 'h' && b[0] == 'h')
4968                         instr->type = INSTR_JMP_LT_MH;
4969                 if (a[0] == 'h' && b[0] == 'h')
4970                         instr->type = INSTR_JMP_LT_HH;
4971                 instr->jmp.ip = NULL; /* Resolved later. */
4972
4973                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4974                 instr->jmp.a.n_bits = fa->n_bits;
4975                 instr->jmp.a.offset = fa->offset / 8;
4976                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4977                 instr->jmp.b.n_bits = fb->n_bits;
4978                 instr->jmp.b.offset = fb->offset / 8;
4979                 return 0;
4980         }
4981
4982         /* JMP_LT_MI, JMP_LT_HI. */
4983         b_val = strtoull(b, &b, 0);
4984         CHECK(!b[0], EINVAL);
4985
4986         instr->type = INSTR_JMP_LT_MI;
4987         if (a[0] == 'h')
4988                 instr->type = INSTR_JMP_LT_HI;
4989         instr->jmp.ip = NULL; /* Resolved later. */
4990
4991         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4992         instr->jmp.a.n_bits = fa->n_bits;
4993         instr->jmp.a.offset = fa->offset / 8;
4994         instr->jmp.b_val = b_val;
4995         return 0;
4996 }
4997
4998 static int
4999 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5000                        struct action *action,
5001                        char **tokens,
5002                        int n_tokens,
5003                        struct instruction *instr,
5004                        struct instruction_data *data)
5005 {
5006         char *a = tokens[2], *b = tokens[3];
5007         struct field *fa, *fb;
5008         uint64_t b_val;
5009         uint32_t a_struct_id, b_struct_id;
5010
5011         CHECK(n_tokens == 4, EINVAL);
5012
5013         strcpy(data->jmp_label, tokens[1]);
5014
5015         fa = struct_field_parse(p, action, a, &a_struct_id);
5016         CHECK(fa, EINVAL);
5017         CHECK(!fa->var_size, EINVAL);
5018
5019         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5020         fb = struct_field_parse(p, action, b, &b_struct_id);
5021         if (fb) {
5022                 CHECK(!fb->var_size, EINVAL);
5023
5024                 instr->type = INSTR_JMP_GT;
5025                 if (a[0] == 'h' && b[0] != 'h')
5026                         instr->type = INSTR_JMP_GT_HM;
5027                 if (a[0] != 'h' && b[0] == 'h')
5028                         instr->type = INSTR_JMP_GT_MH;
5029                 if (a[0] == 'h' && b[0] == 'h')
5030                         instr->type = INSTR_JMP_GT_HH;
5031                 instr->jmp.ip = NULL; /* Resolved later. */
5032
5033                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5034                 instr->jmp.a.n_bits = fa->n_bits;
5035                 instr->jmp.a.offset = fa->offset / 8;
5036                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5037                 instr->jmp.b.n_bits = fb->n_bits;
5038                 instr->jmp.b.offset = fb->offset / 8;
5039                 return 0;
5040         }
5041
5042         /* JMP_GT_MI, JMP_GT_HI. */
5043         b_val = strtoull(b, &b, 0);
5044         CHECK(!b[0], EINVAL);
5045
5046         instr->type = INSTR_JMP_GT_MI;
5047         if (a[0] == 'h')
5048                 instr->type = INSTR_JMP_GT_HI;
5049         instr->jmp.ip = NULL; /* Resolved later. */
5050
5051         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5052         instr->jmp.a.n_bits = fa->n_bits;
5053         instr->jmp.a.offset = fa->offset / 8;
5054         instr->jmp.b_val = b_val;
5055         return 0;
5056 }
5057
5058 static inline void
5059 instr_jmp_exec(struct rte_swx_pipeline *p)
5060 {
5061         struct thread *t = &p->threads[p->thread_id];
5062         struct instruction *ip = t->ip;
5063
5064         TRACE("[Thread %2u] jmp\n", p->thread_id);
5065
5066         thread_ip_set(t, ip->jmp.ip);
5067 }
5068
5069 static inline void
5070 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5071 {
5072         struct thread *t = &p->threads[p->thread_id];
5073         struct instruction *ip = t->ip;
5074         uint32_t header_id = ip->jmp.header_id;
5075
5076         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5077
5078         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5079 }
5080
5081 static inline void
5082 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5083 {
5084         struct thread *t = &p->threads[p->thread_id];
5085         struct instruction *ip = t->ip;
5086         uint32_t header_id = ip->jmp.header_id;
5087
5088         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5089
5090         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5091 }
5092
5093 static inline void
5094 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5095 {
5096         struct thread *t = &p->threads[p->thread_id];
5097         struct instruction *ip = t->ip;
5098         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5099
5100         TRACE("[Thread %2u] jmph\n", p->thread_id);
5101
5102         t->ip = ip_next[t->hit];
5103 }
5104
5105 static inline void
5106 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5107 {
5108         struct thread *t = &p->threads[p->thread_id];
5109         struct instruction *ip = t->ip;
5110         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5111
5112         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5113
5114         t->ip = ip_next[t->hit];
5115 }
5116
5117 static inline void
5118 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5119 {
5120         struct thread *t = &p->threads[p->thread_id];
5121         struct instruction *ip = t->ip;
5122
5123         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5124
5125         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5126 }
5127
5128 static inline void
5129 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5130 {
5131         struct thread *t = &p->threads[p->thread_id];
5132         struct instruction *ip = t->ip;
5133
5134         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5135
5136         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5137 }
5138
5139 static inline void
5140 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5141 {
5142         struct thread *t = &p->threads[p->thread_id];
5143         struct instruction *ip = t->ip;
5144
5145         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5146
5147         JMP_CMP(t, ip, ==);
5148 }
5149
5150 static inline void
5151 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5152 {
5153         struct thread *t = &p->threads[p->thread_id];
5154         struct instruction *ip = t->ip;
5155
5156         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5157
5158         JMP_CMP_MH(t, ip, ==);
5159 }
5160
5161 static inline void
5162 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5163 {
5164         struct thread *t = &p->threads[p->thread_id];
5165         struct instruction *ip = t->ip;
5166
5167         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5168
5169         JMP_CMP_HM(t, ip, ==);
5170 }
5171
5172 static inline void
5173 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5174 {
5175         struct thread *t = &p->threads[p->thread_id];
5176         struct instruction *ip = t->ip;
5177
5178         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5179
5180         JMP_CMP_HH_FAST(t, ip, ==);
5181 }
5182
5183 static inline void
5184 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5185 {
5186         struct thread *t = &p->threads[p->thread_id];
5187         struct instruction *ip = t->ip;
5188
5189         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5190
5191         JMP_CMP_I(t, ip, ==);
5192 }
5193
5194 static inline void
5195 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5196 {
5197         struct thread *t = &p->threads[p->thread_id];
5198         struct instruction *ip = t->ip;
5199
5200         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5201
5202         JMP_CMP(t, ip, !=);
5203 }
5204
5205 static inline void
5206 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5207 {
5208         struct thread *t = &p->threads[p->thread_id];
5209         struct instruction *ip = t->ip;
5210
5211         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5212
5213         JMP_CMP_MH(t, ip, !=);
5214 }
5215
5216 static inline void
5217 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5218 {
5219         struct thread *t = &p->threads[p->thread_id];
5220         struct instruction *ip = t->ip;
5221
5222         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5223
5224         JMP_CMP_HM(t, ip, !=);
5225 }
5226
5227 static inline void
5228 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5229 {
5230         struct thread *t = &p->threads[p->thread_id];
5231         struct instruction *ip = t->ip;
5232
5233         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5234
5235         JMP_CMP_HH_FAST(t, ip, !=);
5236 }
5237
5238 static inline void
5239 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5240 {
5241         struct thread *t = &p->threads[p->thread_id];
5242         struct instruction *ip = t->ip;
5243
5244         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5245
5246         JMP_CMP_I(t, ip, !=);
5247 }
5248
5249 static inline void
5250 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5251 {
5252         struct thread *t = &p->threads[p->thread_id];
5253         struct instruction *ip = t->ip;
5254
5255         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5256
5257         JMP_CMP(t, ip, <);
5258 }
5259
5260 static inline void
5261 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5262 {
5263         struct thread *t = &p->threads[p->thread_id];
5264         struct instruction *ip = t->ip;
5265
5266         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5267
5268         JMP_CMP_MH(t, ip, <);
5269 }
5270
5271 static inline void
5272 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5273 {
5274         struct thread *t = &p->threads[p->thread_id];
5275         struct instruction *ip = t->ip;
5276
5277         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5278
5279         JMP_CMP_HM(t, ip, <);
5280 }
5281
5282 static inline void
5283 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5284 {
5285         struct thread *t = &p->threads[p->thread_id];
5286         struct instruction *ip = t->ip;
5287
5288         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5289
5290         JMP_CMP_HH(t, ip, <);
5291 }
5292
5293 static inline void
5294 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5295 {
5296         struct thread *t = &p->threads[p->thread_id];
5297         struct instruction *ip = t->ip;
5298
5299         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5300
5301         JMP_CMP_MI(t, ip, <);
5302 }
5303
5304 static inline void
5305 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5306 {
5307         struct thread *t = &p->threads[p->thread_id];
5308         struct instruction *ip = t->ip;
5309
5310         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5311
5312         JMP_CMP_HI(t, ip, <);
5313 }
5314
5315 static inline void
5316 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5317 {
5318         struct thread *t = &p->threads[p->thread_id];
5319         struct instruction *ip = t->ip;
5320
5321         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5322
5323         JMP_CMP(t, ip, >);
5324 }
5325
5326 static inline void
5327 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5328 {
5329         struct thread *t = &p->threads[p->thread_id];
5330         struct instruction *ip = t->ip;
5331
5332         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5333
5334         JMP_CMP_MH(t, ip, >);
5335 }
5336
5337 static inline void
5338 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5339 {
5340         struct thread *t = &p->threads[p->thread_id];
5341         struct instruction *ip = t->ip;
5342
5343         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5344
5345         JMP_CMP_HM(t, ip, >);
5346 }
5347
5348 static inline void
5349 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5350 {
5351         struct thread *t = &p->threads[p->thread_id];
5352         struct instruction *ip = t->ip;
5353
5354         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5355
5356         JMP_CMP_HH(t, ip, >);
5357 }
5358
5359 static inline void
5360 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5361 {
5362         struct thread *t = &p->threads[p->thread_id];
5363         struct instruction *ip = t->ip;
5364
5365         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5366
5367         JMP_CMP_MI(t, ip, >);
5368 }
5369
5370 static inline void
5371 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5372 {
5373         struct thread *t = &p->threads[p->thread_id];
5374         struct instruction *ip = t->ip;
5375
5376         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5377
5378         JMP_CMP_HI(t, ip, >);
5379 }
5380
5381 /*
5382  * return.
5383  */
5384 static int
5385 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5386                        struct action *action,
5387                        char **tokens __rte_unused,
5388                        int n_tokens,
5389                        struct instruction *instr,
5390                        struct instruction_data *data __rte_unused)
5391 {
5392         CHECK(action, EINVAL);
5393         CHECK(n_tokens == 1, EINVAL);
5394
5395         instr->type = INSTR_RETURN;
5396         return 0;
5397 }
5398
5399 static inline void
5400 instr_return_exec(struct rte_swx_pipeline *p)
5401 {
5402         struct thread *t = &p->threads[p->thread_id];
5403
5404         TRACE("[Thread %2u] return\n", p->thread_id);
5405
5406         t->ip = t->ret;
5407 }
5408
5409 static int
5410 instr_translate(struct rte_swx_pipeline *p,
5411                 struct action *action,
5412                 char *string,
5413                 struct instruction *instr,
5414                 struct instruction_data *data)
5415 {
5416         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5417         int n_tokens = 0, tpos = 0;
5418
5419         /* Parse the instruction string into tokens. */
5420         for ( ; ; ) {
5421                 char *token;
5422
5423                 token = strtok_r(string, " \t\v", &string);
5424                 if (!token)
5425                         break;
5426
5427                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5428                 CHECK_NAME(token, EINVAL);
5429
5430                 tokens[n_tokens] = token;
5431                 n_tokens++;
5432         }
5433
5434         CHECK(n_tokens, EINVAL);
5435
5436         /* Handle the optional instruction label. */
5437         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5438                 strcpy(data->label, tokens[0]);
5439
5440                 tpos += 2;
5441                 CHECK(n_tokens - tpos, EINVAL);
5442         }
5443
5444         /* Identify the instruction type. */
5445         if (!strcmp(tokens[tpos], "rx"))
5446                 return instr_rx_translate(p,
5447                                           action,
5448                                           &tokens[tpos],
5449                                           n_tokens - tpos,
5450                                           instr,
5451                                           data);
5452
5453         if (!strcmp(tokens[tpos], "tx"))
5454                 return instr_tx_translate(p,
5455                                           action,
5456                                           &tokens[tpos],
5457                                           n_tokens - tpos,
5458                                           instr,
5459                                           data);
5460
5461         if (!strcmp(tokens[tpos], "drop"))
5462                 return instr_drop_translate(p,
5463                                             action,
5464                                             &tokens[tpos],
5465                                             n_tokens - tpos,
5466                                             instr,
5467                                             data);
5468
5469         if (!strcmp(tokens[tpos], "extract"))
5470                 return instr_hdr_extract_translate(p,
5471                                                    action,
5472                                                    &tokens[tpos],
5473                                                    n_tokens - tpos,
5474                                                    instr,
5475                                                    data);
5476
5477         if (!strcmp(tokens[tpos], "lookahead"))
5478                 return instr_hdr_lookahead_translate(p,
5479                                                      action,
5480                                                      &tokens[tpos],
5481                                                      n_tokens - tpos,
5482                                                      instr,
5483                                                      data);
5484
5485         if (!strcmp(tokens[tpos], "emit"))
5486                 return instr_hdr_emit_translate(p,
5487                                                 action,
5488                                                 &tokens[tpos],
5489                                                 n_tokens - tpos,
5490                                                 instr,
5491                                                 data);
5492
5493         if (!strcmp(tokens[tpos], "validate"))
5494                 return instr_hdr_validate_translate(p,
5495                                                     action,
5496                                                     &tokens[tpos],
5497                                                     n_tokens - tpos,
5498                                                     instr,
5499                                                     data);
5500
5501         if (!strcmp(tokens[tpos], "invalidate"))
5502                 return instr_hdr_invalidate_translate(p,
5503                                                       action,
5504                                                       &tokens[tpos],
5505                                                       n_tokens - tpos,
5506                                                       instr,
5507                                                       data);
5508
5509         if (!strcmp(tokens[tpos], "mov"))
5510                 return instr_mov_translate(p,
5511                                            action,
5512                                            &tokens[tpos],
5513                                            n_tokens - tpos,
5514                                            instr,
5515                                            data);
5516
5517         if (!strcmp(tokens[tpos], "add"))
5518                 return instr_alu_add_translate(p,
5519                                                action,
5520                                                &tokens[tpos],
5521                                                n_tokens - tpos,
5522                                                instr,
5523                                                data);
5524
5525         if (!strcmp(tokens[tpos], "sub"))
5526                 return instr_alu_sub_translate(p,
5527                                                action,
5528                                                &tokens[tpos],
5529                                                n_tokens - tpos,
5530                                                instr,
5531                                                data);
5532
5533         if (!strcmp(tokens[tpos], "ckadd"))
5534                 return instr_alu_ckadd_translate(p,
5535                                                  action,
5536                                                  &tokens[tpos],
5537                                                  n_tokens - tpos,
5538                                                  instr,
5539                                                  data);
5540
5541         if (!strcmp(tokens[tpos], "cksub"))
5542                 return instr_alu_cksub_translate(p,
5543                                                  action,
5544                                                  &tokens[tpos],
5545                                                  n_tokens - tpos,
5546                                                  instr,
5547                                                  data);
5548
5549         if (!strcmp(tokens[tpos], "and"))
5550                 return instr_alu_and_translate(p,
5551                                                action,
5552                                                &tokens[tpos],
5553                                                n_tokens - tpos,
5554                                                instr,
5555                                                data);
5556
5557         if (!strcmp(tokens[tpos], "or"))
5558                 return instr_alu_or_translate(p,
5559                                               action,
5560                                               &tokens[tpos],
5561                                               n_tokens - tpos,
5562                                               instr,
5563                                               data);
5564
5565         if (!strcmp(tokens[tpos], "xor"))
5566                 return instr_alu_xor_translate(p,
5567                                                action,
5568                                                &tokens[tpos],
5569                                                n_tokens - tpos,
5570                                                instr,
5571                                                data);
5572
5573         if (!strcmp(tokens[tpos], "shl"))
5574                 return instr_alu_shl_translate(p,
5575                                                action,
5576                                                &tokens[tpos],
5577                                                n_tokens - tpos,
5578                                                instr,
5579                                                data);
5580
5581         if (!strcmp(tokens[tpos], "shr"))
5582                 return instr_alu_shr_translate(p,
5583                                                action,
5584                                                &tokens[tpos],
5585                                                n_tokens - tpos,
5586                                                instr,
5587                                                data);
5588
5589         if (!strcmp(tokens[tpos], "regprefetch"))
5590                 return instr_regprefetch_translate(p,
5591                                                    action,
5592                                                    &tokens[tpos],
5593                                                    n_tokens - tpos,
5594                                                    instr,
5595                                                    data);
5596
5597         if (!strcmp(tokens[tpos], "regrd"))
5598                 return instr_regrd_translate(p,
5599                                              action,
5600                                              &tokens[tpos],
5601                                              n_tokens - tpos,
5602                                              instr,
5603                                              data);
5604
5605         if (!strcmp(tokens[tpos], "regwr"))
5606                 return instr_regwr_translate(p,
5607                                              action,
5608                                              &tokens[tpos],
5609                                              n_tokens - tpos,
5610                                              instr,
5611                                              data);
5612
5613         if (!strcmp(tokens[tpos], "regadd"))
5614                 return instr_regadd_translate(p,
5615                                               action,
5616                                               &tokens[tpos],
5617                                               n_tokens - tpos,
5618                                               instr,
5619                                               data);
5620
5621         if (!strcmp(tokens[tpos], "metprefetch"))
5622                 return instr_metprefetch_translate(p,
5623                                                    action,
5624                                                    &tokens[tpos],
5625                                                    n_tokens - tpos,
5626                                                    instr,
5627                                                    data);
5628
5629         if (!strcmp(tokens[tpos], "meter"))
5630                 return instr_meter_translate(p,
5631                                              action,
5632                                              &tokens[tpos],
5633                                              n_tokens - tpos,
5634                                              instr,
5635                                              data);
5636
5637         if (!strcmp(tokens[tpos], "table"))
5638                 return instr_table_translate(p,
5639                                              action,
5640                                              &tokens[tpos],
5641                                              n_tokens - tpos,
5642                                              instr,
5643                                              data);
5644
5645         if (!strcmp(tokens[tpos], "learn"))
5646                 return instr_learn_translate(p,
5647                                              action,
5648                                              &tokens[tpos],
5649                                              n_tokens - tpos,
5650                                              instr,
5651                                              data);
5652
5653         if (!strcmp(tokens[tpos], "forget"))
5654                 return instr_forget_translate(p,
5655                                               action,
5656                                               &tokens[tpos],
5657                                               n_tokens - tpos,
5658                                               instr,
5659                                               data);
5660
5661         if (!strcmp(tokens[tpos], "extern"))
5662                 return instr_extern_translate(p,
5663                                               action,
5664                                               &tokens[tpos],
5665                                               n_tokens - tpos,
5666                                               instr,
5667                                               data);
5668
5669         if (!strcmp(tokens[tpos], "jmp"))
5670                 return instr_jmp_translate(p,
5671                                            action,
5672                                            &tokens[tpos],
5673                                            n_tokens - tpos,
5674                                            instr,
5675                                            data);
5676
5677         if (!strcmp(tokens[tpos], "jmpv"))
5678                 return instr_jmp_valid_translate(p,
5679                                                  action,
5680                                                  &tokens[tpos],
5681                                                  n_tokens - tpos,
5682                                                  instr,
5683                                                  data);
5684
5685         if (!strcmp(tokens[tpos], "jmpnv"))
5686                 return instr_jmp_invalid_translate(p,
5687                                                    action,
5688                                                    &tokens[tpos],
5689                                                    n_tokens - tpos,
5690                                                    instr,
5691                                                    data);
5692
5693         if (!strcmp(tokens[tpos], "jmph"))
5694                 return instr_jmp_hit_translate(p,
5695                                                action,
5696                                                &tokens[tpos],
5697                                                n_tokens - tpos,
5698                                                instr,
5699                                                data);
5700
5701         if (!strcmp(tokens[tpos], "jmpnh"))
5702                 return instr_jmp_miss_translate(p,
5703                                                 action,
5704                                                 &tokens[tpos],
5705                                                 n_tokens - tpos,
5706                                                 instr,
5707                                                 data);
5708
5709         if (!strcmp(tokens[tpos], "jmpa"))
5710                 return instr_jmp_action_hit_translate(p,
5711                                                       action,
5712                                                       &tokens[tpos],
5713                                                       n_tokens - tpos,
5714                                                       instr,
5715                                                       data);
5716
5717         if (!strcmp(tokens[tpos], "jmpna"))
5718                 return instr_jmp_action_miss_translate(p,
5719                                                        action,
5720                                                        &tokens[tpos],
5721                                                        n_tokens - tpos,
5722                                                        instr,
5723                                                        data);
5724
5725         if (!strcmp(tokens[tpos], "jmpeq"))
5726                 return instr_jmp_eq_translate(p,
5727                                               action,
5728                                               &tokens[tpos],
5729                                               n_tokens - tpos,
5730                                               instr,
5731                                               data);
5732
5733         if (!strcmp(tokens[tpos], "jmpneq"))
5734                 return instr_jmp_neq_translate(p,
5735                                                action,
5736                                                &tokens[tpos],
5737                                                n_tokens - tpos,
5738                                                instr,
5739                                                data);
5740
5741         if (!strcmp(tokens[tpos], "jmplt"))
5742                 return instr_jmp_lt_translate(p,
5743                                               action,
5744                                               &tokens[tpos],
5745                                               n_tokens - tpos,
5746                                               instr,
5747                                               data);
5748
5749         if (!strcmp(tokens[tpos], "jmpgt"))
5750                 return instr_jmp_gt_translate(p,
5751                                               action,
5752                                               &tokens[tpos],
5753                                               n_tokens - tpos,
5754                                               instr,
5755                                               data);
5756
5757         if (!strcmp(tokens[tpos], "return"))
5758                 return instr_return_translate(p,
5759                                               action,
5760                                               &tokens[tpos],
5761                                               n_tokens - tpos,
5762                                               instr,
5763                                               data);
5764
5765         CHECK(0, EINVAL);
5766 }
5767
5768 static struct instruction_data *
5769 label_find(struct instruction_data *data, uint32_t n, const char *label)
5770 {
5771         uint32_t i;
5772
5773         for (i = 0; i < n; i++)
5774                 if (!strcmp(label, data[i].label))
5775                         return &data[i];
5776
5777         return NULL;
5778 }
5779
5780 static uint32_t
5781 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5782 {
5783         uint32_t count = 0, i;
5784
5785         if (!label[0])
5786                 return 0;
5787
5788         for (i = 0; i < n; i++)
5789                 if (!strcmp(label, data[i].jmp_label))
5790                         count++;
5791
5792         return count;
5793 }
5794
5795 static int
5796 instr_label_check(struct instruction_data *instruction_data,
5797                   uint32_t n_instructions)
5798 {
5799         uint32_t i;
5800
5801         /* Check that all instruction labels are unique. */
5802         for (i = 0; i < n_instructions; i++) {
5803                 struct instruction_data *data = &instruction_data[i];
5804                 char *label = data->label;
5805                 uint32_t j;
5806
5807                 if (!label[0])
5808                         continue;
5809
5810                 for (j = i + 1; j < n_instructions; j++)
5811                         CHECK(strcmp(label, data[j].label), EINVAL);
5812         }
5813
5814         /* Get users for each instruction label. */
5815         for (i = 0; i < n_instructions; i++) {
5816                 struct instruction_data *data = &instruction_data[i];
5817                 char *label = data->label;
5818
5819                 data->n_users = label_is_used(instruction_data,
5820                                               n_instructions,
5821                                               label);
5822         }
5823
5824         return 0;
5825 }
5826
5827 static int
5828 instr_jmp_resolve(struct instruction *instructions,
5829                   struct instruction_data *instruction_data,
5830                   uint32_t n_instructions)
5831 {
5832         uint32_t i;
5833
5834         for (i = 0; i < n_instructions; i++) {
5835                 struct instruction *instr = &instructions[i];
5836                 struct instruction_data *data = &instruction_data[i];
5837                 struct instruction_data *found;
5838
5839                 if (!instruction_is_jmp(instr))
5840                         continue;
5841
5842                 found = label_find(instruction_data,
5843                                    n_instructions,
5844                                    data->jmp_label);
5845                 CHECK(found, EINVAL);
5846
5847                 instr->jmp.ip = &instructions[found - instruction_data];
5848         }
5849
5850         return 0;
5851 }
5852
5853 static int
5854 instr_verify(struct rte_swx_pipeline *p __rte_unused,
5855              struct action *a,
5856              struct instruction *instr,
5857              struct instruction_data *data __rte_unused,
5858              uint32_t n_instructions)
5859 {
5860         if (!a) {
5861                 enum instruction_type type;
5862                 uint32_t i;
5863
5864                 /* Check that the first instruction is rx. */
5865                 CHECK(instr[0].type == INSTR_RX, EINVAL);
5866
5867                 /* Check that there is at least one tx instruction. */
5868                 for (i = 0; i < n_instructions; i++) {
5869                         type = instr[i].type;
5870
5871                         if (instruction_is_tx(type))
5872                                 break;
5873                 }
5874                 CHECK(i < n_instructions, EINVAL);
5875
5876                 /* Check that the last instruction is either tx or unconditional
5877                  * jump.
5878                  */
5879                 type = instr[n_instructions - 1].type;
5880                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
5881         }
5882
5883         if (a) {
5884                 enum instruction_type type;
5885                 uint32_t i;
5886
5887                 /* Check that there is at least one return or tx instruction. */
5888                 for (i = 0; i < n_instructions; i++) {
5889                         type = instr[i].type;
5890
5891                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
5892                                 break;
5893                 }
5894                 CHECK(i < n_instructions, EINVAL);
5895         }
5896
5897         return 0;
5898 }
5899
5900 static uint32_t
5901 instr_compact(struct instruction *instructions,
5902               struct instruction_data *instruction_data,
5903               uint32_t n_instructions)
5904 {
5905         uint32_t i, pos = 0;
5906
5907         /* Eliminate the invalid instructions that have been optimized out. */
5908         for (i = 0; i < n_instructions; i++) {
5909                 struct instruction *instr = &instructions[i];
5910                 struct instruction_data *data = &instruction_data[i];
5911
5912                 if (data->invalid)
5913                         continue;
5914
5915                 if (i != pos) {
5916                         memcpy(&instructions[pos], instr, sizeof(*instr));
5917                         memcpy(&instruction_data[pos], data, sizeof(*data));
5918                 }
5919
5920                 pos++;
5921         }
5922
5923         return pos;
5924 }
5925
5926 static int
5927 instr_pattern_extract_many_search(struct instruction *instr,
5928                                   struct instruction_data *data,
5929                                   uint32_t n_instr,
5930                                   uint32_t *n_pattern_instr)
5931 {
5932         uint32_t i;
5933
5934         for (i = 0; i < n_instr; i++) {
5935                 if (data[i].invalid)
5936                         break;
5937
5938                 if (instr[i].type != INSTR_HDR_EXTRACT)
5939                         break;
5940
5941                 if (i == RTE_DIM(instr->io.hdr.header_id))
5942                         break;
5943
5944                 if (i && data[i].n_users)
5945                         break;
5946         }
5947
5948         if (i < 2)
5949                 return 0;
5950
5951         *n_pattern_instr = i;
5952         return 1;
5953 }
5954
5955 static void
5956 instr_pattern_extract_many_replace(struct instruction *instr,
5957                                    struct instruction_data *data,
5958                                    uint32_t n_instr)
5959 {
5960         uint32_t i;
5961
5962         for (i = 1; i < n_instr; i++) {
5963                 instr[0].type++;
5964                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
5965                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
5966                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
5967
5968                 data[i].invalid = 1;
5969         }
5970 }
5971
5972 static uint32_t
5973 instr_pattern_extract_many_optimize(struct instruction *instructions,
5974                                     struct instruction_data *instruction_data,
5975                                     uint32_t n_instructions)
5976 {
5977         uint32_t i;
5978
5979         for (i = 0; i < n_instructions; ) {
5980                 struct instruction *instr = &instructions[i];
5981                 struct instruction_data *data = &instruction_data[i];
5982                 uint32_t n_instr = 0;
5983                 int detected;
5984
5985                 /* Extract many. */
5986                 detected = instr_pattern_extract_many_search(instr,
5987                                                              data,
5988                                                              n_instructions - i,
5989                                                              &n_instr);
5990                 if (detected) {
5991                         instr_pattern_extract_many_replace(instr,
5992                                                            data,
5993                                                            n_instr);
5994                         i += n_instr;
5995                         continue;
5996                 }
5997
5998                 /* No pattern starting at the current instruction. */
5999                 i++;
6000         }
6001
6002         /* Eliminate the invalid instructions that have been optimized out. */
6003         n_instructions = instr_compact(instructions,
6004                                        instruction_data,
6005                                        n_instructions);
6006
6007         return n_instructions;
6008 }
6009
6010 static int
6011 instr_pattern_emit_many_tx_search(struct instruction *instr,
6012                                   struct instruction_data *data,
6013                                   uint32_t n_instr,
6014                                   uint32_t *n_pattern_instr)
6015 {
6016         uint32_t i;
6017
6018         for (i = 0; i < n_instr; i++) {
6019                 if (data[i].invalid)
6020                         break;
6021
6022                 if (instr[i].type != INSTR_HDR_EMIT)
6023                         break;
6024
6025                 if (i == RTE_DIM(instr->io.hdr.header_id))
6026                         break;
6027
6028                 if (i && data[i].n_users)
6029                         break;
6030         }
6031
6032         if (!i)
6033                 return 0;
6034
6035         if (!instruction_is_tx(instr[i].type))
6036                 return 0;
6037
6038         if (data[i].n_users)
6039                 return 0;
6040
6041         i++;
6042
6043         *n_pattern_instr = i;
6044         return 1;
6045 }
6046
6047 static void
6048 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6049                                    struct instruction_data *data,
6050                                    uint32_t n_instr)
6051 {
6052         uint32_t i;
6053
6054         /* Any emit instruction in addition to the first one. */
6055         for (i = 1; i < n_instr - 1; i++) {
6056                 instr[0].type++;
6057                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6058                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6059                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6060
6061                 data[i].invalid = 1;
6062         }
6063
6064         /* The TX instruction is the last one in the pattern. */
6065         instr[0].type++;
6066         instr[0].io.io.offset = instr[i].io.io.offset;
6067         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6068         data[i].invalid = 1;
6069 }
6070
6071 static uint32_t
6072 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6073                                     struct instruction_data *instruction_data,
6074                                     uint32_t n_instructions)
6075 {
6076         uint32_t i;
6077
6078         for (i = 0; i < n_instructions; ) {
6079                 struct instruction *instr = &instructions[i];
6080                 struct instruction_data *data = &instruction_data[i];
6081                 uint32_t n_instr = 0;
6082                 int detected;
6083
6084                 /* Emit many + TX. */
6085                 detected = instr_pattern_emit_many_tx_search(instr,
6086                                                              data,
6087                                                              n_instructions - i,
6088                                                              &n_instr);
6089                 if (detected) {
6090                         instr_pattern_emit_many_tx_replace(instr,
6091                                                            data,
6092                                                            n_instr);
6093                         i += n_instr;
6094                         continue;
6095                 }
6096
6097                 /* No pattern starting at the current instruction. */
6098                 i++;
6099         }
6100
6101         /* Eliminate the invalid instructions that have been optimized out. */
6102         n_instructions = instr_compact(instructions,
6103                                        instruction_data,
6104                                        n_instructions);
6105
6106         return n_instructions;
6107 }
6108
6109 static uint32_t
6110 action_arg_src_mov_count(struct action *a,
6111                          uint32_t arg_id,
6112                          struct instruction *instructions,
6113                          struct instruction_data *instruction_data,
6114                          uint32_t n_instructions);
6115
6116 static int
6117 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6118                                       struct action *a,
6119                                       struct instruction *instr,
6120                                       struct instruction_data *data,
6121                                       uint32_t n_instr,
6122                                       struct instruction *instructions,
6123                                       struct instruction_data *instruction_data,
6124                                       uint32_t n_instructions,
6125                                       uint32_t *n_pattern_instr)
6126 {
6127         struct header *h;
6128         uint32_t src_field_id, i, j;
6129
6130         /* Prerequisites. */
6131         if (!a || !a->st)
6132                 return 0;
6133
6134         /* First instruction: MOV_HM. */
6135         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6136                 return 0;
6137
6138         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6139         if (!h || h->st->var_size)
6140                 return 0;
6141
6142         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6143                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6144                         break;
6145
6146         if (src_field_id == a->st->n_fields)
6147                 return 0;
6148
6149         if (instr[0].mov.dst.offset ||
6150             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6151             instr[0].mov.src.struct_id ||
6152             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6153             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6154                 return 0;
6155
6156         if ((n_instr < h->st->n_fields + 1) ||
6157              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6158                 return 0;
6159
6160         /* Subsequent instructions: MOV_HM. */
6161         for (i = 1; i < h->st->n_fields; i++)
6162                 if (data[i].invalid ||
6163                     data[i].n_users ||
6164                     (instr[i].type != INSTR_MOV_HM) ||
6165                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6166                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6167                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6168                     instr[i].mov.src.struct_id ||
6169                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6170                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6171                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6172                         return 0;
6173
6174         /* Last instruction: HDR_VALIDATE. */
6175         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6176             (instr[i].valid.header_id != h->id))
6177                 return 0;
6178
6179         /* Check that none of the action args that are used as source for this
6180          * DMA transfer are not used as source in any other mov instruction.
6181          */
6182         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6183                 uint32_t n_users;
6184
6185                 n_users = action_arg_src_mov_count(a,
6186                                                    j,
6187                                                    instructions,
6188                                                    instruction_data,
6189                                                    n_instructions);
6190                 if (n_users > 1)
6191                         return 0;
6192         }
6193
6194         *n_pattern_instr = 1 + i;
6195         return 1;
6196 }
6197
6198 static void
6199 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6200                                        struct action *a,
6201                                        struct instruction *instr,
6202                                        struct instruction_data *data,
6203                                        uint32_t n_instr)
6204 {
6205         struct header *h;
6206         uint32_t src_field_id, src_offset, i;
6207
6208         /* Read from the instructions before they are modified. */
6209         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6210         if (!h)
6211                 return;
6212
6213         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6214                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6215                         break;
6216
6217         if (src_field_id == a->st->n_fields)
6218                 return;
6219
6220         src_offset = instr[0].mov.src.offset;
6221
6222         /* Modify the instructions. */
6223         instr[0].type = INSTR_DMA_HT;
6224         instr[0].dma.dst.header_id[0] = h->id;
6225         instr[0].dma.dst.struct_id[0] = h->struct_id;
6226         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6227         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6228
6229         for (i = 1; i < n_instr; i++)
6230                 data[i].invalid = 1;
6231
6232         /* Update the endianness of the action arguments to header endianness. */
6233         for (i = 0; i < h->st->n_fields; i++)
6234                 a->args_endianness[src_field_id + i] = 1;
6235 }
6236
6237 static uint32_t
6238 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6239                                         struct action *a,
6240                                         struct instruction *instructions,
6241                                         struct instruction_data *instruction_data,
6242                                         uint32_t n_instructions)
6243 {
6244         uint32_t i;
6245
6246         if (!a || !a->st)
6247                 return n_instructions;
6248
6249         for (i = 0; i < n_instructions; ) {
6250                 struct instruction *instr = &instructions[i];
6251                 struct instruction_data *data = &instruction_data[i];
6252                 uint32_t n_instr = 0;
6253                 int detected;
6254
6255                 /* Mov all + validate. */
6256                 detected = instr_pattern_mov_all_validate_search(p,
6257                                                                  a,
6258                                                                  instr,
6259                                                                  data,
6260                                                                  n_instructions - i,
6261                                                                  instructions,
6262                                                                  instruction_data,
6263                                                                  n_instructions,
6264                                                                  &n_instr);
6265                 if (detected) {
6266                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6267                         i += n_instr;
6268                         continue;
6269                 }
6270
6271                 /* No pattern starting at the current instruction. */
6272                 i++;
6273         }
6274
6275         /* Eliminate the invalid instructions that have been optimized out. */
6276         n_instructions = instr_compact(instructions,
6277                                        instruction_data,
6278                                        n_instructions);
6279
6280         return n_instructions;
6281 }
6282
6283 static int
6284 instr_pattern_dma_many_search(struct instruction *instr,
6285                               struct instruction_data *data,
6286                               uint32_t n_instr,
6287                               uint32_t *n_pattern_instr)
6288 {
6289         uint32_t i;
6290
6291         for (i = 0; i < n_instr; i++) {
6292                 if (data[i].invalid)
6293                         break;
6294
6295                 if (instr[i].type != INSTR_DMA_HT)
6296                         break;
6297
6298                 if (i == RTE_DIM(instr->dma.dst.header_id))
6299                         break;
6300
6301                 if (i && data[i].n_users)
6302                         break;
6303         }
6304
6305         if (i < 2)
6306                 return 0;
6307
6308         *n_pattern_instr = i;
6309         return 1;
6310 }
6311
6312 static void
6313 instr_pattern_dma_many_replace(struct instruction *instr,
6314                                struct instruction_data *data,
6315                                uint32_t n_instr)
6316 {
6317         uint32_t i;
6318
6319         for (i = 1; i < n_instr; i++) {
6320                 instr[0].type++;
6321                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6322                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6323                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6324                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6325
6326                 data[i].invalid = 1;
6327         }
6328 }
6329
6330 static uint32_t
6331 instr_pattern_dma_many_optimize(struct instruction *instructions,
6332                struct instruction_data *instruction_data,
6333                uint32_t n_instructions)
6334 {
6335         uint32_t i;
6336
6337         for (i = 0; i < n_instructions; ) {
6338                 struct instruction *instr = &instructions[i];
6339                 struct instruction_data *data = &instruction_data[i];
6340                 uint32_t n_instr = 0;
6341                 int detected;
6342
6343                 /* DMA many. */
6344                 detected = instr_pattern_dma_many_search(instr,
6345                                                          data,
6346                                                          n_instructions - i,
6347                                                          &n_instr);
6348                 if (detected) {
6349                         instr_pattern_dma_many_replace(instr, data, n_instr);
6350                         i += n_instr;
6351                         continue;
6352                 }
6353
6354                 /* No pattern starting at the current instruction. */
6355                 i++;
6356         }
6357
6358         /* Eliminate the invalid instructions that have been optimized out. */
6359         n_instructions = instr_compact(instructions,
6360                                        instruction_data,
6361                                        n_instructions);
6362
6363         return n_instructions;
6364 }
6365
6366 static uint32_t
6367 instr_optimize(struct rte_swx_pipeline *p,
6368                struct action *a,
6369                struct instruction *instructions,
6370                struct instruction_data *instruction_data,
6371                uint32_t n_instructions)
6372 {
6373         /* Extract many. */
6374         n_instructions = instr_pattern_extract_many_optimize(instructions,
6375                                                              instruction_data,
6376                                                              n_instructions);
6377
6378         /* Emit many + TX. */
6379         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6380                                                              instruction_data,
6381                                                              n_instructions);
6382
6383         /* Mov all + validate. */
6384         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6385                                                                  a,
6386                                                                  instructions,
6387                                                                  instruction_data,
6388                                                                  n_instructions);
6389
6390         /* DMA many. */
6391         n_instructions = instr_pattern_dma_many_optimize(instructions,
6392                                                          instruction_data,
6393                                                          n_instructions);
6394
6395         return n_instructions;
6396 }
6397
6398 static int
6399 instruction_config(struct rte_swx_pipeline *p,
6400                    struct action *a,
6401                    const char **instructions,
6402                    uint32_t n_instructions)
6403 {
6404         struct instruction *instr = NULL;
6405         struct instruction_data *data = NULL;
6406         int err = 0;
6407         uint32_t i;
6408
6409         CHECK(n_instructions, EINVAL);
6410         CHECK(instructions, EINVAL);
6411         for (i = 0; i < n_instructions; i++)
6412                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6413
6414         /* Memory allocation. */
6415         instr = calloc(n_instructions, sizeof(struct instruction));
6416         if (!instr) {
6417                 err = -ENOMEM;
6418                 goto error;
6419         }
6420
6421         data = calloc(n_instructions, sizeof(struct instruction_data));
6422         if (!data) {
6423                 err = -ENOMEM;
6424                 goto error;
6425         }
6426
6427         for (i = 0; i < n_instructions; i++) {
6428                 char *string = strdup(instructions[i]);
6429                 if (!string) {
6430                         err = -ENOMEM;
6431                         goto error;
6432                 }
6433
6434                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6435                 if (err) {
6436                         free(string);
6437                         goto error;
6438                 }
6439
6440                 free(string);
6441         }
6442
6443         err = instr_label_check(data, n_instructions);
6444         if (err)
6445                 goto error;
6446
6447         err = instr_verify(p, a, instr, data, n_instructions);
6448         if (err)
6449                 goto error;
6450
6451         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6452
6453         err = instr_jmp_resolve(instr, data, n_instructions);
6454         if (err)
6455                 goto error;
6456
6457         if (a) {
6458                 a->instructions = instr;
6459                 a->instruction_data = data;
6460                 a->n_instructions = n_instructions;
6461         } else {
6462                 p->instructions = instr;
6463                 p->instruction_data = data;
6464                 p->n_instructions = n_instructions;
6465         }
6466
6467         return 0;
6468
6469 error:
6470         free(data);
6471         free(instr);
6472         return err;
6473 }
6474
6475 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
6476
6477 static instr_exec_t instruction_table[] = {
6478         [INSTR_RX] = instr_rx_exec,
6479         [INSTR_TX] = instr_tx_exec,
6480         [INSTR_TX_I] = instr_tx_i_exec,
6481
6482         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6483         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6484         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6485         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6486         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6487         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6488         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6489         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6490         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6491         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6492
6493         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6494         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6495         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6496         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6497         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6498         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6499         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6500         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6501         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6502
6503         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6504         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6505
6506         [INSTR_MOV] = instr_mov_exec,
6507         [INSTR_MOV_MH] = instr_mov_mh_exec,
6508         [INSTR_MOV_HM] = instr_mov_hm_exec,
6509         [INSTR_MOV_HH] = instr_mov_hh_exec,
6510         [INSTR_MOV_I] = instr_mov_i_exec,
6511
6512         [INSTR_DMA_HT] = instr_dma_ht_exec,
6513         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6514         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6515         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6516         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6517         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6518         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6519         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6520
6521         [INSTR_ALU_ADD] = instr_alu_add_exec,
6522         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6523         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6524         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6525         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6526         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6527
6528         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6529         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6530         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6531         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6532         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6533         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6534
6535         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6536         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6537         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6538         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6539
6540         [INSTR_ALU_AND] = instr_alu_and_exec,
6541         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6542         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6543         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6544         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6545
6546         [INSTR_ALU_OR] = instr_alu_or_exec,
6547         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6548         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6549         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6550         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6551
6552         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6553         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6554         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6555         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6556         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6557
6558         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6559         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6560         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6561         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6562         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6563         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6564
6565         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6566         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6567         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6568         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6569         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6570         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6571
6572         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6573         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6574         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6575
6576         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6577         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6578         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6579         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6580         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6581         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6582
6583         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6584         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6585         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6586         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6587         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6588         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6589         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6590         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6591         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6592
6593         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6594         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6595         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6596         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6597         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6598         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6599         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6600         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6601         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6602
6603         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6604         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6605         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6606
6607         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6608         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6609         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6610         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6611         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6612         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6613         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6614         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6615         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6616         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6617         [INSTR_METER_IMM] = instr_meter_imm_exec,
6618         [INSTR_METER_IMI] = instr_meter_imi_exec,
6619
6620         [INSTR_TABLE] = instr_table_exec,
6621         [INSTR_SELECTOR] = instr_selector_exec,
6622         [INSTR_LEARNER] = instr_learner_exec,
6623         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6624         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6625         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6626         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6627
6628         [INSTR_JMP] = instr_jmp_exec,
6629         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6630         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6631         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6632         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6633         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6634         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6635
6636         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6637         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6638         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6639         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6640         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6641
6642         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6643         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6644         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6645         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6646         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6647
6648         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6649         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6650         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6651         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6652         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6653         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6654
6655         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6656         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6657         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6658         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6659         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6660         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6661
6662         [INSTR_RETURN] = instr_return_exec,
6663 };
6664
6665 static inline void
6666 instr_exec(struct rte_swx_pipeline *p)
6667 {
6668         struct thread *t = &p->threads[p->thread_id];
6669         struct instruction *ip = t->ip;
6670         instr_exec_t instr = instruction_table[ip->type];
6671
6672         instr(p);
6673 }
6674
6675 /*
6676  * Action.
6677  */
6678 static struct action *
6679 action_find(struct rte_swx_pipeline *p, const char *name)
6680 {
6681         struct action *elem;
6682
6683         if (!name)
6684                 return NULL;
6685
6686         TAILQ_FOREACH(elem, &p->actions, node)
6687                 if (strcmp(elem->name, name) == 0)
6688                         return elem;
6689
6690         return NULL;
6691 }
6692
6693 static struct action *
6694 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6695 {
6696         struct action *action = NULL;
6697
6698         TAILQ_FOREACH(action, &p->actions, node)
6699                 if (action->id == id)
6700                         return action;
6701
6702         return NULL;
6703 }
6704
6705 static struct field *
6706 action_field_find(struct action *a, const char *name)
6707 {
6708         return a->st ? struct_type_field_find(a->st, name) : NULL;
6709 }
6710
6711 static struct field *
6712 action_field_parse(struct action *action, const char *name)
6713 {
6714         if (name[0] != 't' || name[1] != '.')
6715                 return NULL;
6716
6717         return action_field_find(action, &name[2]);
6718 }
6719
6720 static int
6721 action_has_nbo_args(struct action *a)
6722 {
6723         uint32_t i;
6724
6725         /* Return if the action does not have any args. */
6726         if (!a->st)
6727                 return 0; /* FALSE */
6728
6729         for (i = 0; i < a->st->n_fields; i++)
6730                 if (a->args_endianness[i])
6731                         return 1; /* TRUE */
6732
6733         return 0; /* FALSE */
6734 }
6735
6736 static int
6737 action_does_learning(struct action *a)
6738 {
6739         uint32_t i;
6740
6741         for (i = 0; i < a->n_instructions; i++)
6742                 switch (a->instructions[i].type) {
6743                 case INSTR_LEARNER_LEARN:
6744                         return 1; /* TRUE */
6745
6746                 case INSTR_LEARNER_FORGET:
6747                         return 1; /* TRUE */
6748
6749                 default:
6750                         continue;
6751                 }
6752
6753         return 0; /* FALSE */
6754 }
6755
6756 int
6757 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6758                                const char *name,
6759                                const char *args_struct_type_name,
6760                                const char **instructions,
6761                                uint32_t n_instructions)
6762 {
6763         struct struct_type *args_struct_type = NULL;
6764         struct action *a;
6765         int err;
6766
6767         CHECK(p, EINVAL);
6768
6769         CHECK_NAME(name, EINVAL);
6770         CHECK(!action_find(p, name), EEXIST);
6771
6772         if (args_struct_type_name) {
6773                 CHECK_NAME(args_struct_type_name, EINVAL);
6774                 args_struct_type = struct_type_find(p, args_struct_type_name);
6775                 CHECK(args_struct_type, EINVAL);
6776                 CHECK(!args_struct_type->var_size, EINVAL);
6777         }
6778
6779         /* Node allocation. */
6780         a = calloc(1, sizeof(struct action));
6781         CHECK(a, ENOMEM);
6782         if (args_struct_type) {
6783                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6784                 if (!a->args_endianness) {
6785                         free(a);
6786                         CHECK(0, ENOMEM);
6787                 }
6788         }
6789
6790         /* Node initialization. */
6791         strcpy(a->name, name);
6792         a->st = args_struct_type;
6793         a->id = p->n_actions;
6794
6795         /* Instruction translation. */
6796         err = instruction_config(p, a, instructions, n_instructions);
6797         if (err) {
6798                 free(a->args_endianness);
6799                 free(a);
6800                 return err;
6801         }
6802
6803         /* Node add to tailq. */
6804         TAILQ_INSERT_TAIL(&p->actions, a, node);
6805         p->n_actions++;
6806
6807         return 0;
6808 }
6809
6810 static int
6811 action_build(struct rte_swx_pipeline *p)
6812 {
6813         struct action *action;
6814
6815         /* p->action_instructions. */
6816         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
6817         CHECK(p->action_instructions, ENOMEM);
6818
6819         TAILQ_FOREACH(action, &p->actions, node)
6820                 p->action_instructions[action->id] = action->instructions;
6821
6822         return 0;
6823 }
6824
6825 static void
6826 action_build_free(struct rte_swx_pipeline *p)
6827 {
6828         free(p->action_instructions);
6829         p->action_instructions = NULL;
6830 }
6831
6832 static void
6833 action_free(struct rte_swx_pipeline *p)
6834 {
6835         action_build_free(p);
6836
6837         for ( ; ; ) {
6838                 struct action *action;
6839
6840                 action = TAILQ_FIRST(&p->actions);
6841                 if (!action)
6842                         break;
6843
6844                 TAILQ_REMOVE(&p->actions, action, node);
6845                 free(action->instruction_data);
6846                 free(action->instructions);
6847                 free(action);
6848         }
6849 }
6850
6851 static uint32_t
6852 action_arg_src_mov_count(struct action *a,
6853                          uint32_t arg_id,
6854                          struct instruction *instructions,
6855                          struct instruction_data *instruction_data,
6856                          uint32_t n_instructions)
6857 {
6858         uint32_t offset, n_users = 0, i;
6859
6860         if (!a->st ||
6861             (arg_id >= a->st->n_fields) ||
6862             !instructions ||
6863             !instruction_data ||
6864             !n_instructions)
6865                 return 0;
6866
6867         offset = a->st->fields[arg_id].offset / 8;
6868
6869         for (i = 0; i < n_instructions; i++) {
6870                 struct instruction *instr = &instructions[i];
6871                 struct instruction_data *data = &instruction_data[i];
6872
6873                 if (data->invalid ||
6874                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
6875                     instr->mov.src.struct_id ||
6876                     (instr->mov.src.offset != offset))
6877                         continue;
6878
6879                 n_users++;
6880         }
6881
6882         return n_users;
6883 }
6884
6885 /*
6886  * Table.
6887  */
6888 static struct table_type *
6889 table_type_find(struct rte_swx_pipeline *p, const char *name)
6890 {
6891         struct table_type *elem;
6892
6893         TAILQ_FOREACH(elem, &p->table_types, node)
6894                 if (strcmp(elem->name, name) == 0)
6895                         return elem;
6896
6897         return NULL;
6898 }
6899
6900 static struct table_type *
6901 table_type_resolve(struct rte_swx_pipeline *p,
6902                    const char *recommended_type_name,
6903                    enum rte_swx_table_match_type match_type)
6904 {
6905         struct table_type *elem;
6906
6907         /* Only consider the recommended type if the match type is correct. */
6908         if (recommended_type_name)
6909                 TAILQ_FOREACH(elem, &p->table_types, node)
6910                         if (!strcmp(elem->name, recommended_type_name) &&
6911                             (elem->match_type == match_type))
6912                                 return elem;
6913
6914         /* Ignore the recommended type and get the first element with this match
6915          * type.
6916          */
6917         TAILQ_FOREACH(elem, &p->table_types, node)
6918                 if (elem->match_type == match_type)
6919                         return elem;
6920
6921         return NULL;
6922 }
6923
6924 static struct table *
6925 table_find(struct rte_swx_pipeline *p, const char *name)
6926 {
6927         struct table *elem;
6928
6929         TAILQ_FOREACH(elem, &p->tables, node)
6930                 if (strcmp(elem->name, name) == 0)
6931                         return elem;
6932
6933         return NULL;
6934 }
6935
6936 static struct table *
6937 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6938 {
6939         struct table *table = NULL;
6940
6941         TAILQ_FOREACH(table, &p->tables, node)
6942                 if (table->id == id)
6943                         return table;
6944
6945         return NULL;
6946 }
6947
6948 int
6949 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
6950                                      const char *name,
6951                                      enum rte_swx_table_match_type match_type,
6952                                      struct rte_swx_table_ops *ops)
6953 {
6954         struct table_type *elem;
6955
6956         CHECK(p, EINVAL);
6957
6958         CHECK_NAME(name, EINVAL);
6959         CHECK(!table_type_find(p, name), EEXIST);
6960
6961         CHECK(ops, EINVAL);
6962         CHECK(ops->create, EINVAL);
6963         CHECK(ops->lkp, EINVAL);
6964         CHECK(ops->free, EINVAL);
6965
6966         /* Node allocation. */
6967         elem = calloc(1, sizeof(struct table_type));
6968         CHECK(elem, ENOMEM);
6969
6970         /* Node initialization. */
6971         strcpy(elem->name, name);
6972         elem->match_type = match_type;
6973         memcpy(&elem->ops, ops, sizeof(*ops));
6974
6975         /* Node add to tailq. */
6976         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
6977
6978         return 0;
6979 }
6980
6981 static int
6982 table_match_type_resolve(struct rte_swx_match_field_params *fields,
6983                          uint32_t n_fields,
6984                          enum rte_swx_table_match_type *match_type)
6985 {
6986         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
6987
6988         for (i = 0; i < n_fields; i++) {
6989                 struct rte_swx_match_field_params  *f = &fields[i];
6990
6991                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
6992                         n_fields_em++;
6993
6994                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
6995                         n_fields_lpm++;
6996         }
6997
6998         if ((n_fields_lpm > 1) ||
6999             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7000                 return -EINVAL;
7001
7002         *match_type = (n_fields_em == n_fields) ?
7003                        RTE_SWX_TABLE_MATCH_EXACT :
7004                        RTE_SWX_TABLE_MATCH_WILDCARD;
7005
7006         return 0;
7007 }
7008
7009 static int
7010 table_match_fields_check(struct rte_swx_pipeline *p,
7011                          struct rte_swx_pipeline_table_params *params,
7012                          struct header **header)
7013 {
7014         struct header *h0 = NULL;
7015         struct field *hf, *mf;
7016         uint32_t *offset = NULL, i;
7017         int status = 0;
7018
7019         /* Return if no match fields. */
7020         if (!params->n_fields) {
7021                 if (params->fields) {
7022                         status = -EINVAL;
7023                         goto end;
7024                 }
7025
7026                 if (header)
7027                         *header = NULL;
7028
7029                 return 0;
7030         }
7031
7032         /* Memory allocation. */
7033         offset = calloc(params->n_fields, sizeof(uint32_t));
7034         if (!offset) {
7035                 status = -ENOMEM;
7036                 goto end;
7037         }
7038
7039         /* Check that all the match fields belong to either the same header or
7040          * to the meta-data.
7041          */
7042         hf = header_field_parse(p, params->fields[0].name, &h0);
7043         mf = metadata_field_parse(p, params->fields[0].name);
7044         if ((!hf && !mf) || (hf && hf->var_size)) {
7045                 status = -EINVAL;
7046                 goto end;
7047         }
7048
7049         offset[0] = h0 ? hf->offset : mf->offset;
7050
7051         for (i = 1; i < params->n_fields; i++)
7052                 if (h0) {
7053                         struct header *h;
7054
7055                         hf = header_field_parse(p, params->fields[i].name, &h);
7056                         if (!hf || (h->id != h0->id) || hf->var_size) {
7057                                 status = -EINVAL;
7058                                 goto end;
7059                         }
7060
7061                         offset[i] = hf->offset;
7062                 } else {
7063                         mf = metadata_field_parse(p, params->fields[i].name);
7064                         if (!mf) {
7065                                 status = -EINVAL;
7066                                 goto end;
7067                         }
7068
7069                         offset[i] = mf->offset;
7070                 }
7071
7072         /* Check that there are no duplicated match fields. */
7073         for (i = 0; i < params->n_fields; i++) {
7074                 uint32_t j;
7075
7076                 for (j = 0; j < i; j++)
7077                         if (offset[j] == offset[i]) {
7078                                 status = -EINVAL;
7079                                 goto end;
7080                         }
7081         }
7082
7083         /* Return. */
7084         if (header)
7085                 *header = h0;
7086
7087 end:
7088         free(offset);
7089         return status;
7090 }
7091
7092 int
7093 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7094                               const char *name,
7095                               struct rte_swx_pipeline_table_params *params,
7096                               const char *recommended_table_type_name,
7097                               const char *args,
7098                               uint32_t size)
7099 {
7100         struct table_type *type;
7101         struct table *t;
7102         struct action *default_action;
7103         struct header *header = NULL;
7104         uint32_t action_data_size_max = 0, i;
7105         int status = 0;
7106
7107         CHECK(p, EINVAL);
7108
7109         CHECK_NAME(name, EINVAL);
7110         CHECK(!table_find(p, name), EEXIST);
7111         CHECK(!selector_find(p, name), EEXIST);
7112         CHECK(!learner_find(p, name), EEXIST);
7113
7114         CHECK(params, EINVAL);
7115
7116         /* Match checks. */
7117         status = table_match_fields_check(p, params, &header);
7118         if (status)
7119                 return status;
7120
7121         /* Action checks. */
7122         CHECK(params->n_actions, EINVAL);
7123         CHECK(params->action_names, EINVAL);
7124         for (i = 0; i < params->n_actions; i++) {
7125                 const char *action_name = params->action_names[i];
7126                 struct action *a;
7127                 uint32_t action_data_size;
7128
7129                 CHECK_NAME(action_name, EINVAL);
7130
7131                 a = action_find(p, action_name);
7132                 CHECK(a, EINVAL);
7133                 CHECK(!action_does_learning(a), EINVAL);
7134
7135                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7136                 if (action_data_size > action_data_size_max)
7137                         action_data_size_max = action_data_size;
7138         }
7139
7140         CHECK_NAME(params->default_action_name, EINVAL);
7141         for (i = 0; i < p->n_actions; i++)
7142                 if (!strcmp(params->action_names[i],
7143                             params->default_action_name))
7144                         break;
7145         CHECK(i < params->n_actions, EINVAL);
7146         default_action = action_find(p, params->default_action_name);
7147         CHECK((default_action->st && params->default_action_data) ||
7148               !params->default_action_data, EINVAL);
7149
7150         /* Table type checks. */
7151         if (recommended_table_type_name)
7152                 CHECK_NAME(recommended_table_type_name, EINVAL);
7153
7154         if (params->n_fields) {
7155                 enum rte_swx_table_match_type match_type;
7156
7157                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7158                 if (status)
7159                         return status;
7160
7161                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7162                 CHECK(type, EINVAL);
7163         } else {
7164                 type = NULL;
7165         }
7166
7167         /* Memory allocation. */
7168         t = calloc(1, sizeof(struct table));
7169         CHECK(t, ENOMEM);
7170
7171         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7172         if (!t->fields) {
7173                 free(t);
7174                 CHECK(0, ENOMEM);
7175         }
7176
7177         t->actions = calloc(params->n_actions, sizeof(struct action *));
7178         if (!t->actions) {
7179                 free(t->fields);
7180                 free(t);
7181                 CHECK(0, ENOMEM);
7182         }
7183
7184         if (action_data_size_max) {
7185                 t->default_action_data = calloc(1, action_data_size_max);
7186                 if (!t->default_action_data) {
7187                         free(t->actions);
7188                         free(t->fields);
7189                         free(t);
7190                         CHECK(0, ENOMEM);
7191                 }
7192         }
7193
7194         /* Node initialization. */
7195         strcpy(t->name, name);
7196         if (args && args[0])
7197                 strcpy(t->args, args);
7198         t->type = type;
7199
7200         for (i = 0; i < params->n_fields; i++) {
7201                 struct rte_swx_match_field_params *field = &params->fields[i];
7202                 struct match_field *f = &t->fields[i];
7203
7204                 f->match_type = field->match_type;
7205                 f->field = header ?
7206                         header_field_parse(p, field->name, NULL) :
7207                         metadata_field_parse(p, field->name);
7208         }
7209         t->n_fields = params->n_fields;
7210         t->header = header;
7211
7212         for (i = 0; i < params->n_actions; i++)
7213                 t->actions[i] = action_find(p, params->action_names[i]);
7214         t->default_action = default_action;
7215         if (default_action->st)
7216                 memcpy(t->default_action_data,
7217                        params->default_action_data,
7218                        default_action->st->n_bits / 8);
7219         t->n_actions = params->n_actions;
7220         t->default_action_is_const = params->default_action_is_const;
7221         t->action_data_size_max = action_data_size_max;
7222
7223         t->size = size;
7224         t->id = p->n_tables;
7225
7226         /* Node add to tailq. */
7227         TAILQ_INSERT_TAIL(&p->tables, t, node);
7228         p->n_tables++;
7229
7230         return 0;
7231 }
7232
7233 static struct rte_swx_table_params *
7234 table_params_get(struct table *table)
7235 {
7236         struct rte_swx_table_params *params;
7237         struct field *first, *last;
7238         uint8_t *key_mask;
7239         uint32_t key_size, key_offset, action_data_size, i;
7240
7241         /* Memory allocation. */
7242         params = calloc(1, sizeof(struct rte_swx_table_params));
7243         if (!params)
7244                 return NULL;
7245
7246         /* Find first (smallest offset) and last (biggest offset) match fields. */
7247         first = table->fields[0].field;
7248         last = table->fields[0].field;
7249
7250         for (i = 0; i < table->n_fields; i++) {
7251                 struct field *f = table->fields[i].field;
7252
7253                 if (f->offset < first->offset)
7254                         first = f;
7255
7256                 if (f->offset > last->offset)
7257                         last = f;
7258         }
7259
7260         /* Key offset and size. */
7261         key_offset = first->offset / 8;
7262         key_size = (last->offset + last->n_bits - first->offset) / 8;
7263
7264         /* Memory allocation. */
7265         key_mask = calloc(1, key_size);
7266         if (!key_mask) {
7267                 free(params);
7268                 return NULL;
7269         }
7270
7271         /* Key mask. */
7272         for (i = 0; i < table->n_fields; i++) {
7273                 struct field *f = table->fields[i].field;
7274                 uint32_t start = (f->offset - first->offset) / 8;
7275                 size_t size = f->n_bits / 8;
7276
7277                 memset(&key_mask[start], 0xFF, size);
7278         }
7279
7280         /* Action data size. */
7281         action_data_size = 0;
7282         for (i = 0; i < table->n_actions; i++) {
7283                 struct action *action = table->actions[i];
7284                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7285
7286                 if (ads > action_data_size)
7287                         action_data_size = ads;
7288         }
7289
7290         /* Fill in. */
7291         params->match_type = table->type->match_type;
7292         params->key_size = key_size;
7293         params->key_offset = key_offset;
7294         params->key_mask0 = key_mask;
7295         params->action_data_size = action_data_size;
7296         params->n_keys_max = table->size;
7297
7298         return params;
7299 }
7300
7301 static void
7302 table_params_free(struct rte_swx_table_params *params)
7303 {
7304         if (!params)
7305                 return;
7306
7307         free(params->key_mask0);
7308         free(params);
7309 }
7310
7311 static int
7312 table_stub_lkp(void *table __rte_unused,
7313                void *mailbox __rte_unused,
7314                uint8_t **key __rte_unused,
7315                uint64_t *action_id __rte_unused,
7316                uint8_t **action_data __rte_unused,
7317                int *hit)
7318 {
7319         *hit = 0;
7320         return 1; /* DONE. */
7321 }
7322
7323 static int
7324 table_build(struct rte_swx_pipeline *p)
7325 {
7326         uint32_t i;
7327
7328         /* Per pipeline: table statistics. */
7329         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7330         CHECK(p->table_stats, ENOMEM);
7331
7332         for (i = 0; i < p->n_tables; i++) {
7333                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7334                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7335         }
7336
7337         /* Per thread: table runt-time. */
7338         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7339                 struct thread *t = &p->threads[i];
7340                 struct table *table;
7341
7342                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7343                 CHECK(t->tables, ENOMEM);
7344
7345                 TAILQ_FOREACH(table, &p->tables, node) {
7346                         struct table_runtime *r = &t->tables[table->id];
7347
7348                         if (table->type) {
7349                                 uint64_t size;
7350
7351                                 size = table->type->ops.mailbox_size_get();
7352
7353                                 /* r->func. */
7354                                 r->func = table->type->ops.lkp;
7355
7356                                 /* r->mailbox. */
7357                                 if (size) {
7358                                         r->mailbox = calloc(1, size);
7359                                         CHECK(r->mailbox, ENOMEM);
7360                                 }
7361
7362                                 /* r->key. */
7363                                 r->key = table->header ?
7364                                         &t->structs[table->header->struct_id] :
7365                                         &t->structs[p->metadata_struct_id];
7366                         } else {
7367                                 r->func = table_stub_lkp;
7368                         }
7369                 }
7370         }
7371
7372         return 0;
7373 }
7374
7375 static void
7376 table_build_free(struct rte_swx_pipeline *p)
7377 {
7378         uint32_t i;
7379
7380         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7381                 struct thread *t = &p->threads[i];
7382                 uint32_t j;
7383
7384                 if (!t->tables)
7385                         continue;
7386
7387                 for (j = 0; j < p->n_tables; j++) {
7388                         struct table_runtime *r = &t->tables[j];
7389
7390                         free(r->mailbox);
7391                 }
7392
7393                 free(t->tables);
7394                 t->tables = NULL;
7395         }
7396
7397         if (p->table_stats) {
7398                 for (i = 0; i < p->n_tables; i++)
7399                         free(p->table_stats[i].n_pkts_action);
7400
7401                 free(p->table_stats);
7402         }
7403 }
7404
7405 static void
7406 table_free(struct rte_swx_pipeline *p)
7407 {
7408         table_build_free(p);
7409
7410         /* Tables. */
7411         for ( ; ; ) {
7412                 struct table *elem;
7413
7414                 elem = TAILQ_FIRST(&p->tables);
7415                 if (!elem)
7416                         break;
7417
7418                 TAILQ_REMOVE(&p->tables, elem, node);
7419                 free(elem->fields);
7420                 free(elem->actions);
7421                 free(elem->default_action_data);
7422                 free(elem);
7423         }
7424
7425         /* Table types. */
7426         for ( ; ; ) {
7427                 struct table_type *elem;
7428
7429                 elem = TAILQ_FIRST(&p->table_types);
7430                 if (!elem)
7431                         break;
7432
7433                 TAILQ_REMOVE(&p->table_types, elem, node);
7434                 free(elem);
7435         }
7436 }
7437
7438 /*
7439  * Selector.
7440  */
7441 static struct selector *
7442 selector_find(struct rte_swx_pipeline *p, const char *name)
7443 {
7444         struct selector *s;
7445
7446         TAILQ_FOREACH(s, &p->selectors, node)
7447                 if (strcmp(s->name, name) == 0)
7448                         return s;
7449
7450         return NULL;
7451 }
7452
7453 static struct selector *
7454 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7455 {
7456         struct selector *s = NULL;
7457
7458         TAILQ_FOREACH(s, &p->selectors, node)
7459                 if (s->id == id)
7460                         return s;
7461
7462         return NULL;
7463 }
7464
7465 static int
7466 selector_fields_check(struct rte_swx_pipeline *p,
7467                       struct rte_swx_pipeline_selector_params *params,
7468                       struct header **header)
7469 {
7470         struct header *h0 = NULL;
7471         struct field *hf, *mf;
7472         uint32_t i;
7473
7474         /* Return if no selector fields. */
7475         if (!params->n_selector_fields || !params->selector_field_names)
7476                 return -EINVAL;
7477
7478         /* Check that all the selector fields either belong to the same header
7479          * or are all meta-data fields.
7480          */
7481         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7482         mf = metadata_field_parse(p, params->selector_field_names[0]);
7483         if (!hf && !mf)
7484                 return -EINVAL;
7485
7486         for (i = 1; i < params->n_selector_fields; i++)
7487                 if (h0) {
7488                         struct header *h;
7489
7490                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7491                         if (!hf || (h->id != h0->id))
7492                                 return -EINVAL;
7493                 } else {
7494                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7495                         if (!mf)
7496                                 return -EINVAL;
7497                 }
7498
7499         /* Check that there are no duplicated match fields. */
7500         for (i = 0; i < params->n_selector_fields; i++) {
7501                 const char *field_name = params->selector_field_names[i];
7502                 uint32_t j;
7503
7504                 for (j = i + 1; j < params->n_selector_fields; j++)
7505                         if (!strcmp(params->selector_field_names[j], field_name))
7506                                 return -EINVAL;
7507         }
7508
7509         /* Return. */
7510         if (header)
7511                 *header = h0;
7512
7513         return 0;
7514 }
7515
7516 int
7517 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7518                                  const char *name,
7519                                  struct rte_swx_pipeline_selector_params *params)
7520 {
7521         struct selector *s;
7522         struct header *selector_header = NULL;
7523         struct field *group_id_field, *member_id_field;
7524         uint32_t i;
7525         int status = 0;
7526
7527         CHECK(p, EINVAL);
7528
7529         CHECK_NAME(name, EINVAL);
7530         CHECK(!table_find(p, name), EEXIST);
7531         CHECK(!selector_find(p, name), EEXIST);
7532         CHECK(!learner_find(p, name), EEXIST);
7533
7534         CHECK(params, EINVAL);
7535
7536         CHECK_NAME(params->group_id_field_name, EINVAL);
7537         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7538         CHECK(group_id_field, EINVAL);
7539
7540         for (i = 0; i < params->n_selector_fields; i++) {
7541                 const char *field_name = params->selector_field_names[i];
7542
7543                 CHECK_NAME(field_name, EINVAL);
7544         }
7545         status = selector_fields_check(p, params, &selector_header);
7546         if (status)
7547                 return status;
7548
7549         CHECK_NAME(params->member_id_field_name, EINVAL);
7550         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7551         CHECK(member_id_field, EINVAL);
7552
7553         CHECK(params->n_groups_max, EINVAL);
7554
7555         CHECK(params->n_members_per_group_max, EINVAL);
7556
7557         /* Memory allocation. */
7558         s = calloc(1, sizeof(struct selector));
7559         if (!s) {
7560                 status = -ENOMEM;
7561                 goto error;
7562         }
7563
7564         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7565         if (!s->selector_fields) {
7566                 status = -ENOMEM;
7567                 goto error;
7568         }
7569
7570         /* Node initialization. */
7571         strcpy(s->name, name);
7572
7573         s->group_id_field = group_id_field;
7574
7575         for (i = 0; i < params->n_selector_fields; i++) {
7576                 const char *field_name = params->selector_field_names[i];
7577
7578                 s->selector_fields[i] = selector_header ?
7579                         header_field_parse(p, field_name, NULL) :
7580                         metadata_field_parse(p, field_name);
7581         }
7582
7583         s->n_selector_fields = params->n_selector_fields;
7584
7585         s->selector_header = selector_header;
7586
7587         s->member_id_field = member_id_field;
7588
7589         s->n_groups_max = params->n_groups_max;
7590
7591         s->n_members_per_group_max = params->n_members_per_group_max;
7592
7593         s->id = p->n_selectors;
7594
7595         /* Node add to tailq. */
7596         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7597         p->n_selectors++;
7598
7599         return 0;
7600
7601 error:
7602         if (!s)
7603                 return status;
7604
7605         free(s->selector_fields);
7606
7607         free(s);
7608
7609         return status;
7610 }
7611
7612 static void
7613 selector_params_free(struct rte_swx_table_selector_params *params)
7614 {
7615         if (!params)
7616                 return;
7617
7618         free(params->selector_mask);
7619
7620         free(params);
7621 }
7622
7623 static struct rte_swx_table_selector_params *
7624 selector_table_params_get(struct selector *s)
7625 {
7626         struct rte_swx_table_selector_params *params = NULL;
7627         struct field *first, *last;
7628         uint32_t i;
7629
7630         /* Memory allocation. */
7631         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7632         if (!params)
7633                 goto error;
7634
7635         /* Group ID. */
7636         params->group_id_offset = s->group_id_field->offset / 8;
7637
7638         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7639         first = s->selector_fields[0];
7640         last = s->selector_fields[0];
7641
7642         for (i = 0; i < s->n_selector_fields; i++) {
7643                 struct field *f = s->selector_fields[i];
7644
7645                 if (f->offset < first->offset)
7646                         first = f;
7647
7648                 if (f->offset > last->offset)
7649                         last = f;
7650         }
7651
7652         /* Selector offset and size. */
7653         params->selector_offset = first->offset / 8;
7654         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7655
7656         /* Memory allocation. */
7657         params->selector_mask = calloc(1, params->selector_size);
7658         if (!params->selector_mask)
7659                 goto error;
7660
7661         /* Selector mask. */
7662         for (i = 0; i < s->n_selector_fields; i++) {
7663                 struct field *f = s->selector_fields[i];
7664                 uint32_t start = (f->offset - first->offset) / 8;
7665                 size_t size = f->n_bits / 8;
7666
7667                 memset(&params->selector_mask[start], 0xFF, size);
7668         }
7669
7670         /* Member ID. */
7671         params->member_id_offset = s->member_id_field->offset / 8;
7672
7673         /* Maximum number of groups. */
7674         params->n_groups_max = s->n_groups_max;
7675
7676         /* Maximum number of members per group. */
7677         params->n_members_per_group_max = s->n_members_per_group_max;
7678
7679         return params;
7680
7681 error:
7682         selector_params_free(params);
7683         return NULL;
7684 }
7685
7686 static void
7687 selector_build_free(struct rte_swx_pipeline *p)
7688 {
7689         uint32_t i;
7690
7691         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7692                 struct thread *t = &p->threads[i];
7693                 uint32_t j;
7694
7695                 if (!t->selectors)
7696                         continue;
7697
7698                 for (j = 0; j < p->n_selectors; j++) {
7699                         struct selector_runtime *r = &t->selectors[j];
7700
7701                         free(r->mailbox);
7702                 }
7703
7704                 free(t->selectors);
7705                 t->selectors = NULL;
7706         }
7707
7708         free(p->selector_stats);
7709         p->selector_stats = NULL;
7710 }
7711
7712 static int
7713 selector_build(struct rte_swx_pipeline *p)
7714 {
7715         uint32_t i;
7716         int status = 0;
7717
7718         /* Per pipeline: selector statistics. */
7719         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7720         if (!p->selector_stats) {
7721                 status = -ENOMEM;
7722                 goto error;
7723         }
7724
7725         /* Per thread: selector run-time. */
7726         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7727                 struct thread *t = &p->threads[i];
7728                 struct selector *s;
7729
7730                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7731                 if (!t->selectors) {
7732                         status = -ENOMEM;
7733                         goto error;
7734                 }
7735
7736                 TAILQ_FOREACH(s, &p->selectors, node) {
7737                         struct selector_runtime *r = &t->selectors[s->id];
7738                         uint64_t size;
7739
7740                         /* r->mailbox. */
7741                         size = rte_swx_table_selector_mailbox_size_get();
7742                         if (size) {
7743                                 r->mailbox = calloc(1, size);
7744                                 if (!r->mailbox) {
7745                                         status = -ENOMEM;
7746                                         goto error;
7747                                 }
7748                         }
7749
7750                         /* r->group_id_buffer. */
7751                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7752
7753                         /* r->selector_buffer. */
7754                         r->selector_buffer = s->selector_header ?
7755                                 &t->structs[s->selector_header->struct_id] :
7756                                 &t->structs[p->metadata_struct_id];
7757
7758                         /* r->member_id_buffer. */
7759                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
7760                 }
7761         }
7762
7763         return 0;
7764
7765 error:
7766         selector_build_free(p);
7767         return status;
7768 }
7769
7770 static void
7771 selector_free(struct rte_swx_pipeline *p)
7772 {
7773         selector_build_free(p);
7774
7775         /* Selector tables. */
7776         for ( ; ; ) {
7777                 struct selector *elem;
7778
7779                 elem = TAILQ_FIRST(&p->selectors);
7780                 if (!elem)
7781                         break;
7782
7783                 TAILQ_REMOVE(&p->selectors, elem, node);
7784                 free(elem->selector_fields);
7785                 free(elem);
7786         }
7787 }
7788
7789 /*
7790  * Learner table.
7791  */
7792 static struct learner *
7793 learner_find(struct rte_swx_pipeline *p, const char *name)
7794 {
7795         struct learner *l;
7796
7797         TAILQ_FOREACH(l, &p->learners, node)
7798                 if (!strcmp(l->name, name))
7799                         return l;
7800
7801         return NULL;
7802 }
7803
7804 static struct learner *
7805 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7806 {
7807         struct learner *l = NULL;
7808
7809         TAILQ_FOREACH(l, &p->learners, node)
7810                 if (l->id == id)
7811                         return l;
7812
7813         return NULL;
7814 }
7815
7816 static int
7817 learner_match_fields_check(struct rte_swx_pipeline *p,
7818                            struct rte_swx_pipeline_learner_params *params,
7819                            struct header **header)
7820 {
7821         struct header *h0 = NULL;
7822         struct field *hf, *mf;
7823         uint32_t i;
7824
7825         /* Return if no match fields. */
7826         if (!params->n_fields || !params->field_names)
7827                 return -EINVAL;
7828
7829         /* Check that all the match fields either belong to the same header
7830          * or are all meta-data fields.
7831          */
7832         hf = header_field_parse(p, params->field_names[0], &h0);
7833         mf = metadata_field_parse(p, params->field_names[0]);
7834         if (!hf && !mf)
7835                 return -EINVAL;
7836
7837         for (i = 1; i < params->n_fields; i++)
7838                 if (h0) {
7839                         struct header *h;
7840
7841                         hf = header_field_parse(p, params->field_names[i], &h);
7842                         if (!hf || (h->id != h0->id))
7843                                 return -EINVAL;
7844                 } else {
7845                         mf = metadata_field_parse(p, params->field_names[i]);
7846                         if (!mf)
7847                                 return -EINVAL;
7848                 }
7849
7850         /* Check that there are no duplicated match fields. */
7851         for (i = 0; i < params->n_fields; i++) {
7852                 const char *field_name = params->field_names[i];
7853                 uint32_t j;
7854
7855                 for (j = i + 1; j < params->n_fields; j++)
7856                         if (!strcmp(params->field_names[j], field_name))
7857                                 return -EINVAL;
7858         }
7859
7860         /* Return. */
7861         if (header)
7862                 *header = h0;
7863
7864         return 0;
7865 }
7866
7867 static int
7868 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
7869 {
7870         struct struct_type *mst = p->metadata_st, *ast = a->st;
7871         struct field *mf, *af;
7872         uint32_t mf_pos, i;
7873
7874         if (!ast) {
7875                 if (mf_name)
7876                         return -EINVAL;
7877
7878                 return 0;
7879         }
7880
7881         /* Check that mf_name is the name of a valid meta-data field. */
7882         CHECK_NAME(mf_name, EINVAL);
7883         mf = metadata_field_parse(p, mf_name);
7884         CHECK(mf, EINVAL);
7885
7886         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
7887          * all the action arguments.
7888          */
7889         mf_pos = mf - mst->fields;
7890         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
7891
7892         /* Check that the size of each of the identified meta-data fields matches exactly the size
7893          * of the corresponding action argument.
7894          */
7895         for (i = 0; i < ast->n_fields; i++) {
7896                 mf = &mst->fields[mf_pos + i];
7897                 af = &ast->fields[i];
7898
7899                 CHECK(mf->n_bits == af->n_bits, EINVAL);
7900         }
7901
7902         return 0;
7903 }
7904
7905 static int
7906 learner_action_learning_check(struct rte_swx_pipeline *p,
7907                               struct action *action,
7908                               const char **action_names,
7909                               uint32_t n_actions)
7910 {
7911         uint32_t i;
7912
7913         /* For each "learn" instruction of the current action, check that the learned action (i.e.
7914          * the action passed as argument to the "learn" instruction) is also enabled for the
7915          * current learner table.
7916          */
7917         for (i = 0; i < action->n_instructions; i++) {
7918                 struct instruction *instr = &action->instructions[i];
7919                 uint32_t found = 0, j;
7920
7921                 if (instr->type != INSTR_LEARNER_LEARN)
7922                         continue;
7923
7924                 for (j = 0; j < n_actions; j++) {
7925                         struct action *a;
7926
7927                         a = action_find(p, action_names[j]);
7928                         if (!a)
7929                                 return -EINVAL;
7930
7931                         if (a->id == instr->learn.action_id)
7932                                 found = 1;
7933                 }
7934
7935                 if (!found)
7936                         return -EINVAL;
7937         }
7938
7939         return 0;
7940 }
7941
7942 int
7943 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
7944                               const char *name,
7945                               struct rte_swx_pipeline_learner_params *params,
7946                               uint32_t size,
7947                               uint32_t timeout)
7948 {
7949         struct learner *l = NULL;
7950         struct action *default_action;
7951         struct header *header = NULL;
7952         uint32_t action_data_size_max = 0, i;
7953         int status = 0;
7954
7955         CHECK(p, EINVAL);
7956
7957         CHECK_NAME(name, EINVAL);
7958         CHECK(!table_find(p, name), EEXIST);
7959         CHECK(!selector_find(p, name), EEXIST);
7960         CHECK(!learner_find(p, name), EEXIST);
7961
7962         CHECK(params, EINVAL);
7963
7964         /* Match checks. */
7965         status = learner_match_fields_check(p, params, &header);
7966         if (status)
7967                 return status;
7968
7969         /* Action checks. */
7970         CHECK(params->n_actions, EINVAL);
7971
7972         CHECK(params->action_names, EINVAL);
7973         for (i = 0; i < params->n_actions; i++) {
7974                 const char *action_name = params->action_names[i];
7975                 const char *action_field_name = params->action_field_names[i];
7976                 struct action *a;
7977                 uint32_t action_data_size;
7978
7979                 CHECK_NAME(action_name, EINVAL);
7980
7981                 a = action_find(p, action_name);
7982                 CHECK(a, EINVAL);
7983
7984                 status = learner_action_args_check(p, a, action_field_name);
7985                 if (status)
7986                         return status;
7987
7988                 status = learner_action_learning_check(p,
7989                                                        a,
7990                                                        params->action_names,
7991                                                        params->n_actions);
7992                 if (status)
7993                         return status;
7994
7995                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7996                 if (action_data_size > action_data_size_max)
7997                         action_data_size_max = action_data_size;
7998         }
7999
8000         CHECK_NAME(params->default_action_name, EINVAL);
8001         for (i = 0; i < p->n_actions; i++)
8002                 if (!strcmp(params->action_names[i],
8003                             params->default_action_name))
8004                         break;
8005         CHECK(i < params->n_actions, EINVAL);
8006
8007         default_action = action_find(p, params->default_action_name);
8008         CHECK((default_action->st && params->default_action_data) ||
8009               !params->default_action_data, EINVAL);
8010
8011         /* Any other checks. */
8012         CHECK(size, EINVAL);
8013         CHECK(timeout, EINVAL);
8014
8015         /* Memory allocation. */
8016         l = calloc(1, sizeof(struct learner));
8017         if (!l)
8018                 goto nomem;
8019
8020         l->fields = calloc(params->n_fields, sizeof(struct field *));
8021         if (!l->fields)
8022                 goto nomem;
8023
8024         l->actions = calloc(params->n_actions, sizeof(struct action *));
8025         if (!l->actions)
8026                 goto nomem;
8027
8028         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8029         if (!l->action_arg)
8030                 goto nomem;
8031
8032         if (action_data_size_max) {
8033                 l->default_action_data = calloc(1, action_data_size_max);
8034                 if (!l->default_action_data)
8035                         goto nomem;
8036         }
8037
8038         /* Node initialization. */
8039         strcpy(l->name, name);
8040
8041         for (i = 0; i < params->n_fields; i++) {
8042                 const char *field_name = params->field_names[i];
8043
8044                 l->fields[i] = header ?
8045                         header_field_parse(p, field_name, NULL) :
8046                         metadata_field_parse(p, field_name);
8047         }
8048
8049         l->n_fields = params->n_fields;
8050
8051         l->header = header;
8052
8053         for (i = 0; i < params->n_actions; i++) {
8054                 const char *mf_name = params->action_field_names[i];
8055
8056                 l->actions[i] = action_find(p, params->action_names[i]);
8057
8058                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8059         }
8060
8061         l->default_action = default_action;
8062
8063         if (default_action->st)
8064                 memcpy(l->default_action_data,
8065                        params->default_action_data,
8066                        default_action->st->n_bits / 8);
8067
8068         l->n_actions = params->n_actions;
8069
8070         l->default_action_is_const = params->default_action_is_const;
8071
8072         l->action_data_size_max = action_data_size_max;
8073
8074         l->size = size;
8075
8076         l->timeout = timeout;
8077
8078         l->id = p->n_learners;
8079
8080         /* Node add to tailq. */
8081         TAILQ_INSERT_TAIL(&p->learners, l, node);
8082         p->n_learners++;
8083
8084         return 0;
8085
8086 nomem:
8087         if (!l)
8088                 return -ENOMEM;
8089
8090         free(l->action_arg);
8091         free(l->actions);
8092         free(l->fields);
8093         free(l);
8094
8095         return -ENOMEM;
8096 }
8097
8098 static void
8099 learner_params_free(struct rte_swx_table_learner_params *params)
8100 {
8101         if (!params)
8102                 return;
8103
8104         free(params->key_mask0);
8105
8106         free(params);
8107 }
8108
8109 static struct rte_swx_table_learner_params *
8110 learner_params_get(struct learner *l)
8111 {
8112         struct rte_swx_table_learner_params *params = NULL;
8113         struct field *first, *last;
8114         uint32_t i;
8115
8116         /* Memory allocation. */
8117         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8118         if (!params)
8119                 goto error;
8120
8121         /* Find first (smallest offset) and last (biggest offset) match fields. */
8122         first = l->fields[0];
8123         last = l->fields[0];
8124
8125         for (i = 0; i < l->n_fields; i++) {
8126                 struct field *f = l->fields[i];
8127
8128                 if (f->offset < first->offset)
8129                         first = f;
8130
8131                 if (f->offset > last->offset)
8132                         last = f;
8133         }
8134
8135         /* Key offset and size. */
8136         params->key_offset = first->offset / 8;
8137         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8138
8139         /* Memory allocation. */
8140         params->key_mask0 = calloc(1, params->key_size);
8141         if (!params->key_mask0)
8142                 goto error;
8143
8144         /* Key mask. */
8145         for (i = 0; i < l->n_fields; i++) {
8146                 struct field *f = l->fields[i];
8147                 uint32_t start = (f->offset - first->offset) / 8;
8148                 size_t size = f->n_bits / 8;
8149
8150                 memset(&params->key_mask0[start], 0xFF, size);
8151         }
8152
8153         /* Action data size. */
8154         params->action_data_size = l->action_data_size_max;
8155
8156         /* Maximum number of keys. */
8157         params->n_keys_max = l->size;
8158
8159         /* Timeout. */
8160         params->key_timeout = l->timeout;
8161
8162         return params;
8163
8164 error:
8165         learner_params_free(params);
8166         return NULL;
8167 }
8168
8169 static void
8170 learner_build_free(struct rte_swx_pipeline *p)
8171 {
8172         uint32_t i;
8173
8174         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8175                 struct thread *t = &p->threads[i];
8176                 uint32_t j;
8177
8178                 if (!t->learners)
8179                         continue;
8180
8181                 for (j = 0; j < p->n_learners; j++) {
8182                         struct learner_runtime *r = &t->learners[j];
8183
8184                         free(r->mailbox);
8185                         free(r->action_data);
8186                 }
8187
8188                 free(t->learners);
8189                 t->learners = NULL;
8190         }
8191
8192         if (p->learner_stats) {
8193                 for (i = 0; i < p->n_learners; i++)
8194                         free(p->learner_stats[i].n_pkts_action);
8195
8196                 free(p->learner_stats);
8197         }
8198 }
8199
8200 static int
8201 learner_build(struct rte_swx_pipeline *p)
8202 {
8203         uint32_t i;
8204         int status = 0;
8205
8206         /* Per pipeline: learner statistics. */
8207         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8208         CHECK(p->learner_stats, ENOMEM);
8209
8210         for (i = 0; i < p->n_learners; i++) {
8211                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8212                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8213         }
8214
8215         /* Per thread: learner run-time. */
8216         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8217                 struct thread *t = &p->threads[i];
8218                 struct learner *l;
8219
8220                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8221                 if (!t->learners) {
8222                         status = -ENOMEM;
8223                         goto error;
8224                 }
8225
8226                 TAILQ_FOREACH(l, &p->learners, node) {
8227                         struct learner_runtime *r = &t->learners[l->id];
8228                         uint64_t size;
8229                         uint32_t j;
8230
8231                         /* r->mailbox. */
8232                         size = rte_swx_table_learner_mailbox_size_get();
8233                         if (size) {
8234                                 r->mailbox = calloc(1, size);
8235                                 if (!r->mailbox) {
8236                                         status = -ENOMEM;
8237                                         goto error;
8238                                 }
8239                         }
8240
8241                         /* r->key. */
8242                         r->key = l->header ?
8243                                 &t->structs[l->header->struct_id] :
8244                                 &t->structs[p->metadata_struct_id];
8245
8246                         /* r->action_data. */
8247                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8248                         if (!r->action_data) {
8249                                 status = -ENOMEM;
8250                                 goto error;
8251                         }
8252
8253                         for (j = 0; j < l->n_actions; j++) {
8254                                 struct action *a = l->actions[j];
8255                                 struct field *mf = l->action_arg[j];
8256                                 uint8_t *m = t->structs[p->metadata_struct_id];
8257
8258                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8259                         }
8260                 }
8261         }
8262
8263         return 0;
8264
8265 error:
8266         learner_build_free(p);
8267         return status;
8268 }
8269
8270 static void
8271 learner_free(struct rte_swx_pipeline *p)
8272 {
8273         learner_build_free(p);
8274
8275         /* Learner tables. */
8276         for ( ; ; ) {
8277                 struct learner *l;
8278
8279                 l = TAILQ_FIRST(&p->learners);
8280                 if (!l)
8281                         break;
8282
8283                 TAILQ_REMOVE(&p->learners, l, node);
8284                 free(l->fields);
8285                 free(l->actions);
8286                 free(l->action_arg);
8287                 free(l->default_action_data);
8288                 free(l);
8289         }
8290 }
8291
8292 /*
8293  * Table state.
8294  */
8295 static int
8296 table_state_build(struct rte_swx_pipeline *p)
8297 {
8298         struct table *table;
8299         struct selector *s;
8300         struct learner *l;
8301
8302         p->table_state = calloc(p->n_tables + p->n_selectors,
8303                                 sizeof(struct rte_swx_table_state));
8304         CHECK(p->table_state, ENOMEM);
8305
8306         TAILQ_FOREACH(table, &p->tables, node) {
8307                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8308
8309                 if (table->type) {
8310                         struct rte_swx_table_params *params;
8311
8312                         /* ts->obj. */
8313                         params = table_params_get(table);
8314                         CHECK(params, ENOMEM);
8315
8316                         ts->obj = table->type->ops.create(params,
8317                                 NULL,
8318                                 table->args,
8319                                 p->numa_node);
8320
8321                         table_params_free(params);
8322                         CHECK(ts->obj, ENODEV);
8323                 }
8324
8325                 /* ts->default_action_data. */
8326                 if (table->action_data_size_max) {
8327                         ts->default_action_data =
8328                                 malloc(table->action_data_size_max);
8329                         CHECK(ts->default_action_data, ENOMEM);
8330
8331                         memcpy(ts->default_action_data,
8332                                table->default_action_data,
8333                                table->action_data_size_max);
8334                 }
8335
8336                 /* ts->default_action_id. */
8337                 ts->default_action_id = table->default_action->id;
8338         }
8339
8340         TAILQ_FOREACH(s, &p->selectors, node) {
8341                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8342                 struct rte_swx_table_selector_params *params;
8343
8344                 /* ts->obj. */
8345                 params = selector_table_params_get(s);
8346                 CHECK(params, ENOMEM);
8347
8348                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8349
8350                 selector_params_free(params);
8351                 CHECK(ts->obj, ENODEV);
8352         }
8353
8354         TAILQ_FOREACH(l, &p->learners, node) {
8355                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8356                         p->n_selectors + l->id];
8357                 struct rte_swx_table_learner_params *params;
8358
8359                 /* ts->obj. */
8360                 params = learner_params_get(l);
8361                 CHECK(params, ENOMEM);
8362
8363                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8364                 learner_params_free(params);
8365                 CHECK(ts->obj, ENODEV);
8366
8367                 /* ts->default_action_data. */
8368                 if (l->action_data_size_max) {
8369                         ts->default_action_data = malloc(l->action_data_size_max);
8370                         CHECK(ts->default_action_data, ENOMEM);
8371
8372                         memcpy(ts->default_action_data,
8373                                l->default_action_data,
8374                                l->action_data_size_max);
8375                 }
8376
8377                 /* ts->default_action_id. */
8378                 ts->default_action_id = l->default_action->id;
8379         }
8380
8381         return 0;
8382 }
8383
8384 static void
8385 table_state_build_free(struct rte_swx_pipeline *p)
8386 {
8387         uint32_t i;
8388
8389         if (!p->table_state)
8390                 return;
8391
8392         for (i = 0; i < p->n_tables; i++) {
8393                 struct rte_swx_table_state *ts = &p->table_state[i];
8394                 struct table *table = table_find_by_id(p, i);
8395
8396                 /* ts->obj. */
8397                 if (table->type && ts->obj)
8398                         table->type->ops.free(ts->obj);
8399
8400                 /* ts->default_action_data. */
8401                 free(ts->default_action_data);
8402         }
8403
8404         for (i = 0; i < p->n_selectors; i++) {
8405                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8406
8407                 /* ts->obj. */
8408                 if (ts->obj)
8409                         rte_swx_table_selector_free(ts->obj);
8410         }
8411
8412         for (i = 0; i < p->n_learners; i++) {
8413                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8414
8415                 /* ts->obj. */
8416                 if (ts->obj)
8417                         rte_swx_table_learner_free(ts->obj);
8418
8419                 /* ts->default_action_data. */
8420                 free(ts->default_action_data);
8421         }
8422
8423         free(p->table_state);
8424         p->table_state = NULL;
8425 }
8426
8427 static void
8428 table_state_free(struct rte_swx_pipeline *p)
8429 {
8430         table_state_build_free(p);
8431 }
8432
8433 /*
8434  * Register array.
8435  */
8436 static struct regarray *
8437 regarray_find(struct rte_swx_pipeline *p, const char *name)
8438 {
8439         struct regarray *elem;
8440
8441         TAILQ_FOREACH(elem, &p->regarrays, node)
8442                 if (!strcmp(elem->name, name))
8443                         return elem;
8444
8445         return NULL;
8446 }
8447
8448 static struct regarray *
8449 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8450 {
8451         struct regarray *elem = NULL;
8452
8453         TAILQ_FOREACH(elem, &p->regarrays, node)
8454                 if (elem->id == id)
8455                         return elem;
8456
8457         return NULL;
8458 }
8459
8460 int
8461 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8462                               const char *name,
8463                               uint32_t size,
8464                               uint64_t init_val)
8465 {
8466         struct regarray *r;
8467
8468         CHECK(p, EINVAL);
8469
8470         CHECK_NAME(name, EINVAL);
8471         CHECK(!regarray_find(p, name), EEXIST);
8472
8473         CHECK(size, EINVAL);
8474         size = rte_align32pow2(size);
8475
8476         /* Memory allocation. */
8477         r = calloc(1, sizeof(struct regarray));
8478         CHECK(r, ENOMEM);
8479
8480         /* Node initialization. */
8481         strcpy(r->name, name);
8482         r->init_val = init_val;
8483         r->size = size;
8484         r->id = p->n_regarrays;
8485
8486         /* Node add to tailq. */
8487         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8488         p->n_regarrays++;
8489
8490         return 0;
8491 }
8492
8493 static int
8494 regarray_build(struct rte_swx_pipeline *p)
8495 {
8496         struct regarray *regarray;
8497
8498         if (!p->n_regarrays)
8499                 return 0;
8500
8501         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8502         CHECK(p->regarray_runtime, ENOMEM);
8503
8504         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8505                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8506                 uint32_t i;
8507
8508                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8509                                          RTE_CACHE_LINE_SIZE,
8510                                          p->numa_node);
8511                 CHECK(r->regarray, ENOMEM);
8512
8513                 if (regarray->init_val)
8514                         for (i = 0; i < regarray->size; i++)
8515                                 r->regarray[i] = regarray->init_val;
8516
8517                 r->size_mask = regarray->size - 1;
8518         }
8519
8520         return 0;
8521 }
8522
8523 static void
8524 regarray_build_free(struct rte_swx_pipeline *p)
8525 {
8526         uint32_t i;
8527
8528         if (!p->regarray_runtime)
8529                 return;
8530
8531         for (i = 0; i < p->n_regarrays; i++) {
8532                 struct regarray *regarray = regarray_find_by_id(p, i);
8533                 struct regarray_runtime *r = &p->regarray_runtime[i];
8534
8535                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8536         }
8537
8538         free(p->regarray_runtime);
8539         p->regarray_runtime = NULL;
8540 }
8541
8542 static void
8543 regarray_free(struct rte_swx_pipeline *p)
8544 {
8545         regarray_build_free(p);
8546
8547         for ( ; ; ) {
8548                 struct regarray *elem;
8549
8550                 elem = TAILQ_FIRST(&p->regarrays);
8551                 if (!elem)
8552                         break;
8553
8554                 TAILQ_REMOVE(&p->regarrays, elem, node);
8555                 free(elem);
8556         }
8557 }
8558
8559 /*
8560  * Meter array.
8561  */
8562 static struct meter_profile *
8563 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8564 {
8565         struct meter_profile *elem;
8566
8567         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8568                 if (!strcmp(elem->name, name))
8569                         return elem;
8570
8571         return NULL;
8572 }
8573
8574 static struct metarray *
8575 metarray_find(struct rte_swx_pipeline *p, const char *name)
8576 {
8577         struct metarray *elem;
8578
8579         TAILQ_FOREACH(elem, &p->metarrays, node)
8580                 if (!strcmp(elem->name, name))
8581                         return elem;
8582
8583         return NULL;
8584 }
8585
8586 static struct metarray *
8587 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8588 {
8589         struct metarray *elem = NULL;
8590
8591         TAILQ_FOREACH(elem, &p->metarrays, node)
8592                 if (elem->id == id)
8593                         return elem;
8594
8595         return NULL;
8596 }
8597
8598 int
8599 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8600                                  const char *name,
8601                                  uint32_t size)
8602 {
8603         struct metarray *m;
8604
8605         CHECK(p, EINVAL);
8606
8607         CHECK_NAME(name, EINVAL);
8608         CHECK(!metarray_find(p, name), EEXIST);
8609
8610         CHECK(size, EINVAL);
8611         size = rte_align32pow2(size);
8612
8613         /* Memory allocation. */
8614         m = calloc(1, sizeof(struct metarray));
8615         CHECK(m, ENOMEM);
8616
8617         /* Node initialization. */
8618         strcpy(m->name, name);
8619         m->size = size;
8620         m->id = p->n_metarrays;
8621
8622         /* Node add to tailq. */
8623         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8624         p->n_metarrays++;
8625
8626         return 0;
8627 }
8628
8629 struct meter_profile meter_profile_default = {
8630         .node = {0},
8631         .name = "",
8632         .params = {0},
8633
8634         .profile = {
8635                 .cbs = 10000,
8636                 .pbs = 10000,
8637                 .cir_period = 1,
8638                 .cir_bytes_per_period = 1,
8639                 .pir_period = 1,
8640                 .pir_bytes_per_period = 1,
8641         },
8642
8643         .n_users = 0,
8644 };
8645
8646 static void
8647 meter_init(struct meter *m)
8648 {
8649         memset(m, 0, sizeof(struct meter));
8650         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8651         m->profile = &meter_profile_default;
8652         m->color_mask = RTE_COLOR_GREEN;
8653
8654         meter_profile_default.n_users++;
8655 }
8656
8657 static int
8658 metarray_build(struct rte_swx_pipeline *p)
8659 {
8660         struct metarray *m;
8661
8662         if (!p->n_metarrays)
8663                 return 0;
8664
8665         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8666         CHECK(p->metarray_runtime, ENOMEM);
8667
8668         TAILQ_FOREACH(m, &p->metarrays, node) {
8669                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8670                 uint32_t i;
8671
8672                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8673                                          RTE_CACHE_LINE_SIZE,
8674                                          p->numa_node);
8675                 CHECK(r->metarray, ENOMEM);
8676
8677                 for (i = 0; i < m->size; i++)
8678                         meter_init(&r->metarray[i]);
8679
8680                 r->size_mask = m->size - 1;
8681         }
8682
8683         return 0;
8684 }
8685
8686 static void
8687 metarray_build_free(struct rte_swx_pipeline *p)
8688 {
8689         uint32_t i;
8690
8691         if (!p->metarray_runtime)
8692                 return;
8693
8694         for (i = 0; i < p->n_metarrays; i++) {
8695                 struct metarray *m = metarray_find_by_id(p, i);
8696                 struct metarray_runtime *r = &p->metarray_runtime[i];
8697
8698                 env_free(r->metarray, m->size * sizeof(struct meter));
8699         }
8700
8701         free(p->metarray_runtime);
8702         p->metarray_runtime = NULL;
8703 }
8704
8705 static void
8706 metarray_free(struct rte_swx_pipeline *p)
8707 {
8708         metarray_build_free(p);
8709
8710         /* Meter arrays. */
8711         for ( ; ; ) {
8712                 struct metarray *elem;
8713
8714                 elem = TAILQ_FIRST(&p->metarrays);
8715                 if (!elem)
8716                         break;
8717
8718                 TAILQ_REMOVE(&p->metarrays, elem, node);
8719                 free(elem);
8720         }
8721
8722         /* Meter profiles. */
8723         for ( ; ; ) {
8724                 struct meter_profile *elem;
8725
8726                 elem = TAILQ_FIRST(&p->meter_profiles);
8727                 if (!elem)
8728                         break;
8729
8730                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8731                 free(elem);
8732         }
8733 }
8734
8735 /*
8736  * Pipeline.
8737  */
8738 int
8739 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8740 {
8741         struct rte_swx_pipeline *pipeline;
8742
8743         /* Check input parameters. */
8744         CHECK(p, EINVAL);
8745
8746         /* Memory allocation. */
8747         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8748         CHECK(pipeline, ENOMEM);
8749
8750         /* Initialization. */
8751         TAILQ_INIT(&pipeline->struct_types);
8752         TAILQ_INIT(&pipeline->port_in_types);
8753         TAILQ_INIT(&pipeline->ports_in);
8754         TAILQ_INIT(&pipeline->port_out_types);
8755         TAILQ_INIT(&pipeline->ports_out);
8756         TAILQ_INIT(&pipeline->extern_types);
8757         TAILQ_INIT(&pipeline->extern_objs);
8758         TAILQ_INIT(&pipeline->extern_funcs);
8759         TAILQ_INIT(&pipeline->headers);
8760         TAILQ_INIT(&pipeline->actions);
8761         TAILQ_INIT(&pipeline->table_types);
8762         TAILQ_INIT(&pipeline->tables);
8763         TAILQ_INIT(&pipeline->selectors);
8764         TAILQ_INIT(&pipeline->learners);
8765         TAILQ_INIT(&pipeline->regarrays);
8766         TAILQ_INIT(&pipeline->meter_profiles);
8767         TAILQ_INIT(&pipeline->metarrays);
8768
8769         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8770         pipeline->numa_node = numa_node;
8771
8772         *p = pipeline;
8773         return 0;
8774 }
8775
8776 void
8777 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8778 {
8779         if (!p)
8780                 return;
8781
8782         free(p->instruction_data);
8783         free(p->instructions);
8784
8785         metarray_free(p);
8786         regarray_free(p);
8787         table_state_free(p);
8788         learner_free(p);
8789         selector_free(p);
8790         table_free(p);
8791         action_free(p);
8792         metadata_free(p);
8793         header_free(p);
8794         extern_func_free(p);
8795         extern_obj_free(p);
8796         port_out_free(p);
8797         port_in_free(p);
8798         struct_free(p);
8799
8800         free(p);
8801 }
8802
8803 int
8804 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
8805                                      const char **instructions,
8806                                      uint32_t n_instructions)
8807 {
8808         int err;
8809         uint32_t i;
8810
8811         err = instruction_config(p, NULL, instructions, n_instructions);
8812         if (err)
8813                 return err;
8814
8815         /* Thread instruction pointer reset. */
8816         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8817                 struct thread *t = &p->threads[i];
8818
8819                 thread_ip_reset(p, t);
8820         }
8821
8822         return 0;
8823 }
8824
8825 int
8826 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
8827 {
8828         int status;
8829
8830         CHECK(p, EINVAL);
8831         CHECK(p->build_done == 0, EEXIST);
8832
8833         status = port_in_build(p);
8834         if (status)
8835                 goto error;
8836
8837         status = port_out_build(p);
8838         if (status)
8839                 goto error;
8840
8841         status = struct_build(p);
8842         if (status)
8843                 goto error;
8844
8845         status = extern_obj_build(p);
8846         if (status)
8847                 goto error;
8848
8849         status = extern_func_build(p);
8850         if (status)
8851                 goto error;
8852
8853         status = header_build(p);
8854         if (status)
8855                 goto error;
8856
8857         status = metadata_build(p);
8858         if (status)
8859                 goto error;
8860
8861         status = action_build(p);
8862         if (status)
8863                 goto error;
8864
8865         status = table_build(p);
8866         if (status)
8867                 goto error;
8868
8869         status = selector_build(p);
8870         if (status)
8871                 goto error;
8872
8873         status = learner_build(p);
8874         if (status)
8875                 goto error;
8876
8877         status = table_state_build(p);
8878         if (status)
8879                 goto error;
8880
8881         status = regarray_build(p);
8882         if (status)
8883                 goto error;
8884
8885         status = metarray_build(p);
8886         if (status)
8887                 goto error;
8888
8889         p->build_done = 1;
8890         return 0;
8891
8892 error:
8893         metarray_build_free(p);
8894         regarray_build_free(p);
8895         table_state_build_free(p);
8896         learner_build_free(p);
8897         selector_build_free(p);
8898         table_build_free(p);
8899         action_build_free(p);
8900         metadata_build_free(p);
8901         header_build_free(p);
8902         extern_func_build_free(p);
8903         extern_obj_build_free(p);
8904         port_out_build_free(p);
8905         port_in_build_free(p);
8906         struct_build_free(p);
8907
8908         return status;
8909 }
8910
8911 void
8912 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
8913 {
8914         uint32_t i;
8915
8916         for (i = 0; i < n_instructions; i++)
8917                 instr_exec(p);
8918 }
8919
8920 void
8921 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
8922 {
8923         uint32_t i;
8924
8925         for (i = 0; i < p->n_ports_out; i++) {
8926                 struct port_out_runtime *port = &p->out[i];
8927
8928                 if (port->flush)
8929                         port->flush(port->obj);
8930         }
8931 }
8932
8933 /*
8934  * Control.
8935  */
8936 int
8937 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
8938                               struct rte_swx_ctl_pipeline_info *pipeline)
8939 {
8940         struct action *action;
8941         struct table *table;
8942         uint32_t n_actions = 0, n_tables = 0;
8943
8944         if (!p || !pipeline)
8945                 return -EINVAL;
8946
8947         TAILQ_FOREACH(action, &p->actions, node)
8948                 n_actions++;
8949
8950         TAILQ_FOREACH(table, &p->tables, node)
8951                 n_tables++;
8952
8953         pipeline->n_ports_in = p->n_ports_in;
8954         pipeline->n_ports_out = p->n_ports_out;
8955         pipeline->n_actions = n_actions;
8956         pipeline->n_tables = n_tables;
8957         pipeline->n_selectors = p->n_selectors;
8958         pipeline->n_learners = p->n_learners;
8959         pipeline->n_regarrays = p->n_regarrays;
8960         pipeline->n_metarrays = p->n_metarrays;
8961
8962         return 0;
8963 }
8964
8965 int
8966 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
8967 {
8968         if (!p || !numa_node)
8969                 return -EINVAL;
8970
8971         *numa_node = p->numa_node;
8972         return 0;
8973 }
8974
8975 int
8976 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
8977                             uint32_t action_id,
8978                             struct rte_swx_ctl_action_info *action)
8979 {
8980         struct action *a = NULL;
8981
8982         if (!p || (action_id >= p->n_actions) || !action)
8983                 return -EINVAL;
8984
8985         a = action_find_by_id(p, action_id);
8986         if (!a)
8987                 return -EINVAL;
8988
8989         strcpy(action->name, a->name);
8990         action->n_args = a->st ? a->st->n_fields : 0;
8991         return 0;
8992 }
8993
8994 int
8995 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
8996                                 uint32_t action_id,
8997                                 uint32_t action_arg_id,
8998                                 struct rte_swx_ctl_action_arg_info *action_arg)
8999 {
9000         struct action *a = NULL;
9001         struct field *arg = NULL;
9002
9003         if (!p || (action_id >= p->n_actions) || !action_arg)
9004                 return -EINVAL;
9005
9006         a = action_find_by_id(p, action_id);
9007         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9008                 return -EINVAL;
9009
9010         arg = &a->st->fields[action_arg_id];
9011         strcpy(action_arg->name, arg->name);
9012         action_arg->n_bits = arg->n_bits;
9013         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9014
9015         return 0;
9016 }
9017
9018 int
9019 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9020                            uint32_t table_id,
9021                            struct rte_swx_ctl_table_info *table)
9022 {
9023         struct table *t = NULL;
9024
9025         if (!p || !table)
9026                 return -EINVAL;
9027
9028         t = table_find_by_id(p, table_id);
9029         if (!t)
9030                 return -EINVAL;
9031
9032         strcpy(table->name, t->name);
9033         strcpy(table->args, t->args);
9034         table->n_match_fields = t->n_fields;
9035         table->n_actions = t->n_actions;
9036         table->default_action_is_const = t->default_action_is_const;
9037         table->size = t->size;
9038         return 0;
9039 }
9040
9041 int
9042 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9043         uint32_t table_id,
9044         uint32_t match_field_id,
9045         struct rte_swx_ctl_table_match_field_info *match_field)
9046 {
9047         struct table *t;
9048         struct match_field *f;
9049
9050         if (!p || (table_id >= p->n_tables) || !match_field)
9051                 return -EINVAL;
9052
9053         t = table_find_by_id(p, table_id);
9054         if (!t || (match_field_id >= t->n_fields))
9055                 return -EINVAL;
9056
9057         f = &t->fields[match_field_id];
9058         match_field->match_type = f->match_type;
9059         match_field->is_header = t->header ? 1 : 0;
9060         match_field->n_bits = f->field->n_bits;
9061         match_field->offset = f->field->offset;
9062
9063         return 0;
9064 }
9065
9066 int
9067 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9068         uint32_t table_id,
9069         uint32_t table_action_id,
9070         struct rte_swx_ctl_table_action_info *table_action)
9071 {
9072         struct table *t;
9073
9074         if (!p || (table_id >= p->n_tables) || !table_action)
9075                 return -EINVAL;
9076
9077         t = table_find_by_id(p, table_id);
9078         if (!t || (table_action_id >= t->n_actions))
9079                 return -EINVAL;
9080
9081         table_action->action_id = t->actions[table_action_id]->id;
9082
9083         return 0;
9084 }
9085
9086 int
9087 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9088                           uint32_t table_id,
9089                           struct rte_swx_table_ops *table_ops,
9090                           int *is_stub)
9091 {
9092         struct table *t;
9093
9094         if (!p || (table_id >= p->n_tables))
9095                 return -EINVAL;
9096
9097         t = table_find_by_id(p, table_id);
9098         if (!t)
9099                 return -EINVAL;
9100
9101         if (t->type) {
9102                 if (table_ops)
9103                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9104                 *is_stub = 0;
9105         } else {
9106                 *is_stub = 1;
9107         }
9108
9109         return 0;
9110 }
9111
9112 int
9113 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9114                               uint32_t selector_id,
9115                               struct rte_swx_ctl_selector_info *selector)
9116 {
9117         struct selector *s = NULL;
9118
9119         if (!p || !selector)
9120                 return -EINVAL;
9121
9122         s = selector_find_by_id(p, selector_id);
9123         if (!s)
9124                 return -EINVAL;
9125
9126         strcpy(selector->name, s->name);
9127
9128         selector->n_selector_fields = s->n_selector_fields;
9129         selector->n_groups_max = s->n_groups_max;
9130         selector->n_members_per_group_max = s->n_members_per_group_max;
9131
9132         return 0;
9133 }
9134
9135 int
9136 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9137          uint32_t selector_id,
9138          struct rte_swx_ctl_table_match_field_info *field)
9139 {
9140         struct selector *s;
9141
9142         if (!p || (selector_id >= p->n_selectors) || !field)
9143                 return -EINVAL;
9144
9145         s = selector_find_by_id(p, selector_id);
9146         if (!s)
9147                 return -EINVAL;
9148
9149         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9150         field->is_header = 0;
9151         field->n_bits = s->group_id_field->n_bits;
9152         field->offset = s->group_id_field->offset;
9153
9154         return 0;
9155 }
9156
9157 int
9158 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9159          uint32_t selector_id,
9160          uint32_t selector_field_id,
9161          struct rte_swx_ctl_table_match_field_info *field)
9162 {
9163         struct selector *s;
9164         struct field *f;
9165
9166         if (!p || (selector_id >= p->n_selectors) || !field)
9167                 return -EINVAL;
9168
9169         s = selector_find_by_id(p, selector_id);
9170         if (!s || (selector_field_id >= s->n_selector_fields))
9171                 return -EINVAL;
9172
9173         f = s->selector_fields[selector_field_id];
9174         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9175         field->is_header = s->selector_header ? 1 : 0;
9176         field->n_bits = f->n_bits;
9177         field->offset = f->offset;
9178
9179         return 0;
9180 }
9181
9182 int
9183 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9184          uint32_t selector_id,
9185          struct rte_swx_ctl_table_match_field_info *field)
9186 {
9187         struct selector *s;
9188
9189         if (!p || (selector_id >= p->n_selectors) || !field)
9190                 return -EINVAL;
9191
9192         s = selector_find_by_id(p, selector_id);
9193         if (!s)
9194                 return -EINVAL;
9195
9196         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9197         field->is_header = 0;
9198         field->n_bits = s->member_id_field->n_bits;
9199         field->offset = s->member_id_field->offset;
9200
9201         return 0;
9202 }
9203
9204 int
9205 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9206                              uint32_t learner_id,
9207                              struct rte_swx_ctl_learner_info *learner)
9208 {
9209         struct learner *l = NULL;
9210
9211         if (!p || !learner)
9212                 return -EINVAL;
9213
9214         l = learner_find_by_id(p, learner_id);
9215         if (!l)
9216                 return -EINVAL;
9217
9218         strcpy(learner->name, l->name);
9219
9220         learner->n_match_fields = l->n_fields;
9221         learner->n_actions = l->n_actions;
9222         learner->default_action_is_const = l->default_action_is_const;
9223         learner->size = l->size;
9224
9225         return 0;
9226 }
9227
9228 int
9229 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9230                                          uint32_t learner_id,
9231                                          uint32_t match_field_id,
9232                                          struct rte_swx_ctl_table_match_field_info *match_field)
9233 {
9234         struct learner *l;
9235         struct field *f;
9236
9237         if (!p || (learner_id >= p->n_learners) || !match_field)
9238                 return -EINVAL;
9239
9240         l = learner_find_by_id(p, learner_id);
9241         if (!l || (match_field_id >= l->n_fields))
9242                 return -EINVAL;
9243
9244         f = l->fields[match_field_id];
9245         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9246         match_field->is_header = l->header ? 1 : 0;
9247         match_field->n_bits = f->n_bits;
9248         match_field->offset = f->offset;
9249
9250         return 0;
9251 }
9252
9253 int
9254 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9255                                     uint32_t learner_id,
9256                                     uint32_t learner_action_id,
9257                                     struct rte_swx_ctl_table_action_info *learner_action)
9258 {
9259         struct learner *l;
9260
9261         if (!p || (learner_id >= p->n_learners) || !learner_action)
9262                 return -EINVAL;
9263
9264         l = learner_find_by_id(p, learner_id);
9265         if (!l || (learner_action_id >= l->n_actions))
9266                 return -EINVAL;
9267
9268         learner_action->action_id = l->actions[learner_action_id]->id;
9269
9270         return 0;
9271 }
9272
9273 int
9274 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9275                                  struct rte_swx_table_state **table_state)
9276 {
9277         if (!p || !table_state || !p->build_done)
9278                 return -EINVAL;
9279
9280         *table_state = p->table_state;
9281         return 0;
9282 }
9283
9284 int
9285 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9286                                  struct rte_swx_table_state *table_state)
9287 {
9288         if (!p || !table_state || !p->build_done)
9289                 return -EINVAL;
9290
9291         p->table_state = table_state;
9292         return 0;
9293 }
9294
9295 int
9296 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9297                                         uint32_t port_id,
9298                                         struct rte_swx_port_in_stats *stats)
9299 {
9300         struct port_in *port;
9301
9302         if (!p || !stats)
9303                 return -EINVAL;
9304
9305         port = port_in_find(p, port_id);
9306         if (!port)
9307                 return -EINVAL;
9308
9309         port->type->ops.stats_read(port->obj, stats);
9310         return 0;
9311 }
9312
9313 int
9314 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9315                                          uint32_t port_id,
9316                                          struct rte_swx_port_out_stats *stats)
9317 {
9318         struct port_out *port;
9319
9320         if (!p || !stats)
9321                 return -EINVAL;
9322
9323         port = port_out_find(p, port_id);
9324         if (!port)
9325                 return -EINVAL;
9326
9327         port->type->ops.stats_read(port->obj, stats);
9328         return 0;
9329 }
9330
9331 int
9332 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9333                                       const char *table_name,
9334                                       struct rte_swx_table_stats *stats)
9335 {
9336         struct table *table;
9337         struct table_statistics *table_stats;
9338
9339         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9340                 return -EINVAL;
9341
9342         table = table_find(p, table_name);
9343         if (!table)
9344                 return -EINVAL;
9345
9346         table_stats = &p->table_stats[table->id];
9347
9348         memcpy(stats->n_pkts_action,
9349                table_stats->n_pkts_action,
9350                p->n_actions * sizeof(uint64_t));
9351
9352         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9353         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9354
9355         return 0;
9356 }
9357
9358 int
9359 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9360         const char *selector_name,
9361         struct rte_swx_pipeline_selector_stats *stats)
9362 {
9363         struct selector *s;
9364
9365         if (!p || !selector_name || !selector_name[0] || !stats)
9366                 return -EINVAL;
9367
9368         s = selector_find(p, selector_name);
9369         if (!s)
9370                 return -EINVAL;
9371
9372         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9373
9374         return 0;
9375 }
9376
9377 int
9378 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9379                                         const char *learner_name,
9380                                         struct rte_swx_learner_stats *stats)
9381 {
9382         struct learner *l;
9383         struct learner_statistics *learner_stats;
9384
9385         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9386                 return -EINVAL;
9387
9388         l = learner_find(p, learner_name);
9389         if (!l)
9390                 return -EINVAL;
9391
9392         learner_stats = &p->learner_stats[l->id];
9393
9394         memcpy(stats->n_pkts_action,
9395                learner_stats->n_pkts_action,
9396                p->n_actions * sizeof(uint64_t));
9397
9398         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9399         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9400
9401         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9402         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9403
9404         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9405
9406         return 0;
9407 }
9408
9409 int
9410 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9411                               uint32_t regarray_id,
9412                               struct rte_swx_ctl_regarray_info *regarray)
9413 {
9414         struct regarray *r;
9415
9416         if (!p || !regarray)
9417                 return -EINVAL;
9418
9419         r = regarray_find_by_id(p, regarray_id);
9420         if (!r)
9421                 return -EINVAL;
9422
9423         strcpy(regarray->name, r->name);
9424         regarray->size = r->size;
9425         return 0;
9426 }
9427
9428 int
9429 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9430                                    const char *regarray_name,
9431                                    uint32_t regarray_index,
9432                                    uint64_t *value)
9433 {
9434         struct regarray *regarray;
9435         struct regarray_runtime *r;
9436
9437         if (!p || !regarray_name || !value)
9438                 return -EINVAL;
9439
9440         regarray = regarray_find(p, regarray_name);
9441         if (!regarray || (regarray_index >= regarray->size))
9442                 return -EINVAL;
9443
9444         r = &p->regarray_runtime[regarray->id];
9445         *value = r->regarray[regarray_index];
9446         return 0;
9447 }
9448
9449 int
9450 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9451                                    const char *regarray_name,
9452                                    uint32_t regarray_index,
9453                                    uint64_t value)
9454 {
9455         struct regarray *regarray;
9456         struct regarray_runtime *r;
9457
9458         if (!p || !regarray_name)
9459                 return -EINVAL;
9460
9461         regarray = regarray_find(p, regarray_name);
9462         if (!regarray || (regarray_index >= regarray->size))
9463                 return -EINVAL;
9464
9465         r = &p->regarray_runtime[regarray->id];
9466         r->regarray[regarray_index] = value;
9467         return 0;
9468 }
9469
9470 int
9471 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9472                               uint32_t metarray_id,
9473                               struct rte_swx_ctl_metarray_info *metarray)
9474 {
9475         struct metarray *m;
9476
9477         if (!p || !metarray)
9478                 return -EINVAL;
9479
9480         m = metarray_find_by_id(p, metarray_id);
9481         if (!m)
9482                 return -EINVAL;
9483
9484         strcpy(metarray->name, m->name);
9485         metarray->size = m->size;
9486         return 0;
9487 }
9488
9489 int
9490 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9491                               const char *name,
9492                               struct rte_meter_trtcm_params *params)
9493 {
9494         struct meter_profile *mp;
9495         int status;
9496
9497         CHECK(p, EINVAL);
9498         CHECK_NAME(name, EINVAL);
9499         CHECK(params, EINVAL);
9500         CHECK(!meter_profile_find(p, name), EEXIST);
9501
9502         /* Node allocation. */
9503         mp = calloc(1, sizeof(struct meter_profile));
9504         CHECK(mp, ENOMEM);
9505
9506         /* Node initialization. */
9507         strcpy(mp->name, name);
9508         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9509         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9510         if (status) {
9511                 free(mp);
9512                 CHECK(0, EINVAL);
9513         }
9514
9515         /* Node add to tailq. */
9516         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9517
9518         return 0;
9519 }
9520
9521 int
9522 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9523                                  const char *name)
9524 {
9525         struct meter_profile *mp;
9526
9527         CHECK(p, EINVAL);
9528         CHECK_NAME(name, EINVAL);
9529
9530         mp = meter_profile_find(p, name);
9531         CHECK(mp, EINVAL);
9532         CHECK(!mp->n_users, EBUSY);
9533
9534         /* Remove node from tailq. */
9535         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9536         free(mp);
9537
9538         return 0;
9539 }
9540
9541 int
9542 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9543                         const char *metarray_name,
9544                         uint32_t metarray_index)
9545 {
9546         struct meter_profile *mp_old;
9547         struct metarray *metarray;
9548         struct metarray_runtime *metarray_runtime;
9549         struct meter *m;
9550
9551         CHECK(p, EINVAL);
9552         CHECK_NAME(metarray_name, EINVAL);
9553
9554         metarray = metarray_find(p, metarray_name);
9555         CHECK(metarray, EINVAL);
9556         CHECK(metarray_index < metarray->size, EINVAL);
9557
9558         metarray_runtime = &p->metarray_runtime[metarray->id];
9559         m = &metarray_runtime->metarray[metarray_index];
9560         mp_old = m->profile;
9561
9562         meter_init(m);
9563
9564         mp_old->n_users--;
9565
9566         return 0;
9567 }
9568
9569 int
9570 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9571                       const char *metarray_name,
9572                       uint32_t metarray_index,
9573                       const char *profile_name)
9574 {
9575         struct meter_profile *mp, *mp_old;
9576         struct metarray *metarray;
9577         struct metarray_runtime *metarray_runtime;
9578         struct meter *m;
9579
9580         CHECK(p, EINVAL);
9581         CHECK_NAME(metarray_name, EINVAL);
9582
9583         metarray = metarray_find(p, metarray_name);
9584         CHECK(metarray, EINVAL);
9585         CHECK(metarray_index < metarray->size, EINVAL);
9586
9587         mp = meter_profile_find(p, profile_name);
9588         CHECK(mp, EINVAL);
9589
9590         metarray_runtime = &p->metarray_runtime[metarray->id];
9591         m = &metarray_runtime->metarray[metarray_index];
9592         mp_old = m->profile;
9593
9594         memset(m, 0, sizeof(struct meter));
9595         rte_meter_trtcm_config(&m->m, &mp->profile);
9596         m->profile = mp;
9597         m->color_mask = RTE_COLORS;
9598
9599         mp->n_users++;
9600         mp_old->n_users--;
9601
9602         return 0;
9603 }
9604
9605 int
9606 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9607                              const char *metarray_name,
9608                              uint32_t metarray_index,
9609                              struct rte_swx_ctl_meter_stats *stats)
9610 {
9611         struct metarray *metarray;
9612         struct metarray_runtime *metarray_runtime;
9613         struct meter *m;
9614
9615         CHECK(p, EINVAL);
9616         CHECK_NAME(metarray_name, EINVAL);
9617
9618         metarray = metarray_find(p, metarray_name);
9619         CHECK(metarray, EINVAL);
9620         CHECK(metarray_index < metarray->size, EINVAL);
9621
9622         CHECK(stats, EINVAL);
9623
9624         metarray_runtime = &p->metarray_runtime[metarray->id];
9625         m = &metarray_runtime->metarray[metarray_index];
9626
9627         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9628         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9629
9630         return 0;
9631 }