drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_ip.h>
  12 #include <rte_udp.h>
  13 #include <rte_memzone.h>
  14
  15 #include "enic_compat.h"
  16 #include "enic.h"
  17 #include "vnic_dev.h"
  18 #include "vnic_nic.h"
  19
  20 #define IP_DEFTTL  64   /* from RFC 1340. */
  21 #define IP6_VTC_FLOW 0x60000000
  22
  23 /* Highest Item type supported by Flowman */
  24 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  25
  26 /* Up to 1024 TCAM entries */
  27 #define FM_MAX_TCAM_TABLE_SIZE 1024
  28
  29 /* Up to 4096 entries per exact match table */
  30 #define FM_MAX_EXACT_TABLE_SIZE 4096
  31
  32 /* Number of counters to increase on for each increment */
  33 #define FM_COUNTERS_EXPAND  100
  34
  35 #define FM_INVALID_HANDLE 0
  36
  37 /*
  38  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  39  * Use a simple scheme to map groups to tables.
  40  * Group 0 uses the single TCAM tables, one for each direction.
  41  * Group 1, 2, ... uses its own exact match table.
  42  *
  43  * The TCAM tables are allocated upfront during init.
  44  *
  45  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  46  *
  47  * 1. Add a flow that jumps from group 0 to group N.
  48  *
  49  * If N does not exist, we allocate an exact match table for it, using
  50  * a dummy key. A key is required for the table.
  51  *
  52  * 2. Add a flow that uses group N.
  53  *
  54  * If N does not exist, we allocate an exact match table for it, using
  55  * the flow's key. Subsequent flows to the same group all should have
  56  * the same key.
  57  *
  58  * Without a jump flow to N, N is not reachable in hardware. No packets
  59  * reach N and match.
  60  *
  61  * 3. Add a flow to an empty group N.
  62  *
  63  * N has been created via (1) and the dummy key. We free that table, allocate
  64  * a new table using the new flow's key. Also re-do the existing jump flow to
  65  * point to the new table.
  66  */
  67 #define FM_TCAM_RTE_GROUP 0
  68
  69 struct enic_fm_fet {
  70         TAILQ_ENTRY(enic_fm_fet) list;
  71         uint32_t group; /* rte_flow group ID */
  72         uint64_t handle; /* Exact match table handle from flowman */
  73         uint8_t ingress;
  74         uint8_t default_key;
  75         int ref; /* Reference count via get/put */
  76         struct fm_key_template key; /* Key associated with the table */
  77 };
  78
  79 struct enic_fm_counter {
  80         SLIST_ENTRY(enic_fm_counter) next;
  81         uint32_t handle;
  82 };
  83
  84 /* rte_flow.fm */
  85 struct enic_fm_flow {
  86         bool counter_valid;
  87         uint64_t entry_handle;
  88         uint64_t action_handle;
  89         struct enic_fm_counter *counter;
  90         struct enic_fm_fet *fet;
  91 };
  92
  93 struct enic_fm_jump_flow {
  94         TAILQ_ENTRY(enic_fm_jump_flow) list;
  95         struct rte_flow *flow;
  96         uint32_t group;
  97         struct fm_tcam_match_entry match;
  98         struct fm_action action;
  99 };
 100
 101 /*
 102  * Flowman uses host memory for commands. This structure is allocated
 103  * in DMA-able memory.
 104  */
 105 union enic_flowman_cmd_mem {
 106         struct fm_tcam_match_table fm_tcam_match_table;
 107         struct fm_exact_match_table fm_exact_match_table;
 108         struct fm_tcam_match_entry fm_tcam_match_entry;
 109         struct fm_exact_match_entry fm_exact_match_entry;
 110         struct fm_action fm_action;
 111 };
 112
 113 struct enic_flowman {
 114         struct enic *enic;
 115         /* Command buffer */
 116         struct {
 117                 union enic_flowman_cmd_mem *va;
 118                 dma_addr_t pa;
 119         } cmd;
 120         /* TCAM tables allocated upfront, used for group 0 */
 121         uint64_t ig_tcam_hndl;
 122         uint64_t eg_tcam_hndl;
 123         /* Counters */
 124         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 125         void *counter_stack;
 126         uint32_t counters_alloced;
 127         /* Exact match tables for groups != 0, dynamically allocated */
 128         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 129         /*
 130          * Default exact match tables used for jump actions to
 131          * non-existent groups.
 132          */
 133         struct enic_fm_fet *default_eg_fet;
 134         struct enic_fm_fet *default_ig_fet;
 135         /* Flows that jump to the default table above */
 136         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 137         /*
 138          * Scratch data used during each invocation of flow_create
 139          * and flow_validate.
 140          */
 141         struct enic_fm_fet *fet;
 142         struct fm_tcam_match_entry tcam_entry;
 143         struct fm_action action;
 144         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 145         int action_op_count;
 146 };
 147
 148 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 149
 150 /*
 151  * Common arguments passed to copy_item functions. Use this structure
 152  * so we can easily add new arguments.
 153  * item: Item specification.
 154  * fm_tcam_entry: Flowman TCAM match entry.
 155  * header_level: 0 for outer header, 1 for inner header.
 156  */
 157 struct copy_item_args {
 158         const struct rte_flow_item *item;
 159         struct fm_tcam_match_entry *fm_tcam_entry;
 160         uint8_t header_level;
 161 };
 162
 163 /* functions for copying items into flowman match */
 164 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 165
 166 /* Info about how to copy items into flowman match */
 167 struct enic_fm_items {
 168         /* Function for copying and validating an item. */
 169         enic_copy_item_fn * const copy_item;
 170         /* List of valid previous items. */
 171         const enum rte_flow_item_type * const prev_items;
 172         /*
 173          * True if it's OK for this item to be the first item. For some NIC
 174          * versions, it's invalid to start the stack above layer 3.
 175          */
 176         const uint8_t valid_start_item;
 177 };
 178
 179 static enic_copy_item_fn enic_fm_copy_item_eth;
 180 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 181 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 182 static enic_copy_item_fn enic_fm_copy_item_raw;
 183 static enic_copy_item_fn enic_fm_copy_item_sctp;
 184 static enic_copy_item_fn enic_fm_copy_item_tcp;
 185 static enic_copy_item_fn enic_fm_copy_item_udp;
 186 static enic_copy_item_fn enic_fm_copy_item_vlan;
 187 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 188
 189 /* Ingress actions */
 190 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 191         RTE_FLOW_ACTION_TYPE_COUNT,
 192         RTE_FLOW_ACTION_TYPE_DROP,
 193         RTE_FLOW_ACTION_TYPE_FLAG,
 194         RTE_FLOW_ACTION_TYPE_JUMP,
 195         RTE_FLOW_ACTION_TYPE_MARK,
 196         RTE_FLOW_ACTION_TYPE_PORT_ID,
 197         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 198         RTE_FLOW_ACTION_TYPE_QUEUE,
 199         RTE_FLOW_ACTION_TYPE_RSS,
 200         RTE_FLOW_ACTION_TYPE_VOID,
 201         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 202         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 203         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 204 };
 205
 206 /* Egress actions */
 207 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 208         RTE_FLOW_ACTION_TYPE_COUNT,
 209         RTE_FLOW_ACTION_TYPE_DROP,
 210         RTE_FLOW_ACTION_TYPE_JUMP,
 211         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 212         RTE_FLOW_ACTION_TYPE_VOID,
 213         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 214         RTE_FLOW_ACTION_TYPE_END,
 215 };
 216
 217 static const struct enic_fm_items enic_fm_items[] = {
 218         [RTE_FLOW_ITEM_TYPE_RAW] = {
 219                 .copy_item = enic_fm_copy_item_raw,
 220                 .valid_start_item = 0,
 221                 .prev_items = (const enum rte_flow_item_type[]) {
 222                                RTE_FLOW_ITEM_TYPE_UDP,
 223                                RTE_FLOW_ITEM_TYPE_END,
 224                 },
 225         },
 226         [RTE_FLOW_ITEM_TYPE_ETH] = {
 227                 .copy_item = enic_fm_copy_item_eth,
 228                 .valid_start_item = 1,
 229                 .prev_items = (const enum rte_flow_item_type[]) {
 230                                RTE_FLOW_ITEM_TYPE_END,
 231                 },
 232         },
 233         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 234                 .copy_item = enic_fm_copy_item_vlan,
 235                 .valid_start_item = 1,
 236                 .prev_items = (const enum rte_flow_item_type[]) {
 237                                RTE_FLOW_ITEM_TYPE_ETH,
 238                                RTE_FLOW_ITEM_TYPE_END,
 239                 },
 240         },
 241         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 242                 .copy_item = enic_fm_copy_item_ipv4,
 243                 .valid_start_item = 1,
 244                 .prev_items = (const enum rte_flow_item_type[]) {
 245                                RTE_FLOW_ITEM_TYPE_ETH,
 246                                RTE_FLOW_ITEM_TYPE_VLAN,
 247                                RTE_FLOW_ITEM_TYPE_END,
 248                 },
 249         },
 250         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 251                 .copy_item = enic_fm_copy_item_ipv6,
 252                 .valid_start_item = 1,
 253                 .prev_items = (const enum rte_flow_item_type[]) {
 254                                RTE_FLOW_ITEM_TYPE_ETH,
 255                                RTE_FLOW_ITEM_TYPE_VLAN,
 256                                RTE_FLOW_ITEM_TYPE_END,
 257                 },
 258         },
 259         [RTE_FLOW_ITEM_TYPE_UDP] = {
 260                 .copy_item = enic_fm_copy_item_udp,
 261                 .valid_start_item = 1,
 262                 .prev_items = (const enum rte_flow_item_type[]) {
 263                                RTE_FLOW_ITEM_TYPE_IPV4,
 264                                RTE_FLOW_ITEM_TYPE_IPV6,
 265                                RTE_FLOW_ITEM_TYPE_END,
 266                 },
 267         },
 268         [RTE_FLOW_ITEM_TYPE_TCP] = {
 269                 .copy_item = enic_fm_copy_item_tcp,
 270                 .valid_start_item = 1,
 271                 .prev_items = (const enum rte_flow_item_type[]) {
 272                                RTE_FLOW_ITEM_TYPE_IPV4,
 273                                RTE_FLOW_ITEM_TYPE_IPV6,
 274                                RTE_FLOW_ITEM_TYPE_END,
 275                 },
 276         },
 277         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 278                 .copy_item = enic_fm_copy_item_sctp,
 279                 .valid_start_item = 0,
 280                 .prev_items = (const enum rte_flow_item_type[]) {
 281                                RTE_FLOW_ITEM_TYPE_IPV4,
 282                                RTE_FLOW_ITEM_TYPE_IPV6,
 283                                RTE_FLOW_ITEM_TYPE_END,
 284                 },
 285         },
 286         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 287                 .copy_item = enic_fm_copy_item_vxlan,
 288                 .valid_start_item = 1,
 289                 .prev_items = (const enum rte_flow_item_type[]) {
 290                                RTE_FLOW_ITEM_TYPE_UDP,
 291                                RTE_FLOW_ITEM_TYPE_END,
 292                 },
 293         },
 294 };
 295
 296 static int
 297 enic_fm_copy_item_eth(struct copy_item_args *arg)
 298 {
 299         const struct rte_flow_item *item = arg->item;
 300         const struct rte_flow_item_eth *spec = item->spec;
 301         const struct rte_flow_item_eth *mask = item->mask;
 302         const uint8_t lvl = arg->header_level;
 303         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 304         struct fm_header_set *fm_data, *fm_mask;
 305
 306         ENICPMD_FUNC_TRACE();
 307         /* Match all if no spec */
 308         if (!spec)
 309                 return 0;
 310         if (!mask)
 311                 mask = &rte_flow_item_eth_mask;
 312         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 313         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 314         fm_data->fk_header_select |= FKH_ETHER;
 315         fm_mask->fk_header_select |= FKH_ETHER;
 316         memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
 317         memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
 318         return 0;
 319 }
 320
 321 static int
 322 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 323 {
 324         const struct rte_flow_item *item = arg->item;
 325         const struct rte_flow_item_vlan *spec = item->spec;
 326         const struct rte_flow_item_vlan *mask = item->mask;
 327         const uint8_t lvl = arg->header_level;
 328         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 329         struct fm_header_set *fm_data, *fm_mask;
 330         struct rte_ether_hdr *eth_mask;
 331         struct rte_ether_hdr *eth_val;
 332         uint32_t meta;
 333
 334         ENICPMD_FUNC_TRACE();
 335         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 336         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 337         /* Outer and inner packet vlans need different flags */
 338         meta = FKM_VLAN_PRES;
 339         if (lvl > 0)
 340                 meta = FKM_QTAG;
 341         fm_data->fk_metadata |= meta;
 342         fm_mask->fk_metadata |= meta;
 343
 344         /* Match all if no spec */
 345         if (!spec)
 346                 return 0;
 347         if (!mask)
 348                 mask = &rte_flow_item_vlan_mask;
 349
 350         eth_mask = (void *)&fm_mask->l2.eth;
 351         eth_val = (void *)&fm_data->l2.eth;
 352
 353         /* Outer TPID cannot be matched */
 354         if (eth_mask->ether_type)
 355                 return -ENOTSUP;
 356
 357         /*
 358          * When packet matching, the VIC always compares vlan-stripped
 359          * L2, regardless of vlan stripping settings. So, the inner type
 360          * from vlan becomes the ether type of the eth header.
 361          */
 362         eth_mask->ether_type = mask->inner_type;
 363         eth_val->ether_type = spec->inner_type;
 364         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 365         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 366         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 367         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 368         return 0;
 369 }
 370
 371 static int
 372 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 373 {
 374         const struct rte_flow_item *item = arg->item;
 375         const struct rte_flow_item_ipv4 *spec = item->spec;
 376         const struct rte_flow_item_ipv4 *mask = item->mask;
 377         const uint8_t lvl = arg->header_level;
 378         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 379         struct fm_header_set *fm_data, *fm_mask;
 380
 381         ENICPMD_FUNC_TRACE();
 382         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 383         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 384         fm_data->fk_metadata |= FKM_IPV4;
 385         fm_mask->fk_metadata |= FKM_IPV4;
 386
 387         if (!spec)
 388                 return 0;
 389         if (!mask)
 390                 mask = &rte_flow_item_ipv4_mask;
 391
 392         fm_data->fk_header_select |= FKH_IPV4;
 393         fm_mask->fk_header_select |= FKH_IPV4;
 394         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 395         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 396         return 0;
 397 }
 398
 399 static int
 400 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 401 {
 402         const struct rte_flow_item *item = arg->item;
 403         const struct rte_flow_item_ipv6 *spec = item->spec;
 404         const struct rte_flow_item_ipv6 *mask = item->mask;
 405         const uint8_t lvl = arg->header_level;
 406         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 407         struct fm_header_set *fm_data, *fm_mask;
 408
 409         ENICPMD_FUNC_TRACE();
 410         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 411         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 412         fm_data->fk_metadata |= FKM_IPV6;
 413         fm_mask->fk_metadata |= FKM_IPV6;
 414
 415         if (!spec)
 416                 return 0;
 417         if (!mask)
 418                 mask = &rte_flow_item_ipv6_mask;
 419
 420         fm_data->fk_header_select |= FKH_IPV6;
 421         fm_mask->fk_header_select |= FKH_IPV6;
 422         memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
 423         memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
 424         return 0;
 425 }
 426
 427 static int
 428 enic_fm_copy_item_udp(struct copy_item_args *arg)
 429 {
 430         const struct rte_flow_item *item = arg->item;
 431         const struct rte_flow_item_udp *spec = item->spec;
 432         const struct rte_flow_item_udp *mask = item->mask;
 433         const uint8_t lvl = arg->header_level;
 434         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 435         struct fm_header_set *fm_data, *fm_mask;
 436
 437         ENICPMD_FUNC_TRACE();
 438         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 439         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 440         fm_data->fk_metadata |= FKM_UDP;
 441         fm_mask->fk_metadata |= FKM_UDP;
 442
 443         if (!spec)
 444                 return 0;
 445         if (!mask)
 446                 mask = &rte_flow_item_udp_mask;
 447
 448         fm_data->fk_header_select |= FKH_UDP;
 449         fm_mask->fk_header_select |= FKH_UDP;
 450         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 451         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 452         return 0;
 453 }
 454
 455 static int
 456 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 457 {
 458         const struct rte_flow_item *item = arg->item;
 459         const struct rte_flow_item_tcp *spec = item->spec;
 460         const struct rte_flow_item_tcp *mask = item->mask;
 461         const uint8_t lvl = arg->header_level;
 462         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 463         struct fm_header_set *fm_data, *fm_mask;
 464
 465         ENICPMD_FUNC_TRACE();
 466         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 467         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 468         fm_data->fk_metadata |= FKM_TCP;
 469         fm_mask->fk_metadata |= FKM_TCP;
 470
 471         if (!spec)
 472                 return 0;
 473         if (!mask)
 474                 mask = &rte_flow_item_tcp_mask;
 475
 476         fm_data->fk_header_select |= FKH_TCP;
 477         fm_mask->fk_header_select |= FKH_TCP;
 478         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 479         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 480         return 0;
 481 }
 482
 483 static int
 484 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 485 {
 486         const struct rte_flow_item *item = arg->item;
 487         const struct rte_flow_item_sctp *spec = item->spec;
 488         const struct rte_flow_item_sctp *mask = item->mask;
 489         const uint8_t lvl = arg->header_level;
 490         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 491         struct fm_header_set *fm_data, *fm_mask;
 492         uint8_t *ip_proto_mask = NULL;
 493         uint8_t *ip_proto = NULL;
 494         uint32_t l3_fkh;
 495
 496         ENICPMD_FUNC_TRACE();
 497         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 498         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 499         /*
 500          * The NIC filter API has no flags for "match sctp", so explicitly
 501          * set the protocol number in the IP pattern.
 502          */
 503         if (fm_data->fk_metadata & FKM_IPV4) {
 504                 struct rte_ipv4_hdr *ip;
 505                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 506                 ip_proto_mask = &ip->next_proto_id;
 507                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 508                 ip_proto = &ip->next_proto_id;
 509                 l3_fkh = FKH_IPV4;
 510         } else if (fm_data->fk_metadata & FKM_IPV6) {
 511                 struct rte_ipv6_hdr *ip;
 512                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 513                 ip_proto_mask = &ip->proto;
 514                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 515                 ip_proto = &ip->proto;
 516                 l3_fkh = FKH_IPV6;
 517         } else {
 518                 /* Need IPv4/IPv6 pattern first */
 519                 return -EINVAL;
 520         }
 521         *ip_proto = IPPROTO_SCTP;
 522         *ip_proto_mask = 0xff;
 523         fm_data->fk_header_select |= l3_fkh;
 524         fm_mask->fk_header_select |= l3_fkh;
 525
 526         if (!spec)
 527                 return 0;
 528         if (!mask)
 529                 mask = &rte_flow_item_sctp_mask;
 530
 531         fm_data->fk_header_select |= FKH_L4RAW;
 532         fm_mask->fk_header_select |= FKH_L4RAW;
 533         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 534         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 535         return 0;
 536 }
 537
 538 static int
 539 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 540 {
 541         const struct rte_flow_item *item = arg->item;
 542         const struct rte_flow_item_vxlan *spec = item->spec;
 543         const struct rte_flow_item_vxlan *mask = item->mask;
 544         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 545         struct fm_header_set *fm_data, *fm_mask;
 546
 547         ENICPMD_FUNC_TRACE();
 548         /* Only 2 header levels (outer and inner) allowed */
 549         if (arg->header_level > 0)
 550                 return -EINVAL;
 551
 552         fm_data = &entry->ftm_data.fk_hdrset[0];
 553         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 554         fm_data->fk_metadata |= FKM_VXLAN;
 555         fm_mask->fk_metadata |= FKM_VXLAN;
 556         /* items from here on out are inner header items */
 557         arg->header_level = 1;
 558
 559         /* Match all if no spec */
 560         if (!spec)
 561                 return 0;
 562         if (!mask)
 563                 mask = &rte_flow_item_vxlan_mask;
 564
 565         fm_data->fk_header_select |= FKH_VXLAN;
 566         fm_mask->fk_header_select |= FKH_VXLAN;
 567         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 568         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 569         return 0;
 570 }
 571
 572 /*
 573  * Currently, raw pattern match is very limited. It is intended for matching
 574  * UDP tunnel header (e.g. vxlan or geneve).
 575  */
 576 static int
 577 enic_fm_copy_item_raw(struct copy_item_args *arg)
 578 {
 579         const struct rte_flow_item *item = arg->item;
 580         const struct rte_flow_item_raw *spec = item->spec;
 581         const struct rte_flow_item_raw *mask = item->mask;
 582         const uint8_t lvl = arg->header_level;
 583         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 584         struct fm_header_set *fm_data, *fm_mask;
 585
 586         ENICPMD_FUNC_TRACE();
 587         /* Cannot be used for inner packet */
 588         if (lvl > 0)
 589                 return -EINVAL;
 590         /* Need both spec and mask */
 591         if (!spec || !mask)
 592                 return -EINVAL;
 593         /* Only supports relative with offset 0 */
 594         if (!spec->relative || spec->offset != 0 || spec->search ||
 595             spec->limit)
 596                 return -EINVAL;
 597         /* Need non-null pattern that fits within the NIC's filter pattern */
 598         if (spec->length == 0 ||
 599             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 600             !spec->pattern || !mask->pattern)
 601                 return -EINVAL;
 602         /*
 603          * Mask fields, including length, are often set to zero. Assume that
 604          * means "same as spec" to avoid breaking existing apps. If length
 605          * is not zero, then it should be >= spec length.
 606          *
 607          * No more pattern follows this, so append to the L4 layer instead of
 608          * L5 to work with both recent and older VICs.
 609          */
 610         if (mask->length != 0 && mask->length < spec->length)
 611                 return -EINVAL;
 612
 613         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 614         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 615         fm_data->fk_header_select |= FKH_L4RAW;
 616         fm_mask->fk_header_select |= FKH_L4RAW;
 617         fm_data->fk_header_select &= ~FKH_UDP;
 618         fm_mask->fk_header_select &= ~FKH_UDP;
 619         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 620                spec->pattern, spec->length);
 621         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 622                mask->pattern, spec->length);
 623         return 0;
 624 }
 625
 626 static int
 627 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 628                struct fm_key_template *key, int entries,
 629                struct enic_fm_fet **fet_out)
 630 {
 631         struct fm_exact_match_table *cmd;
 632         struct fm_header_set *hdr;
 633         struct enic_fm_fet *fet;
 634         uint64_t args[3];
 635         int ret;
 636
 637         ENICPMD_FUNC_TRACE();
 638         fet = calloc(1, sizeof(struct enic_fm_fet));
 639         if (fet == NULL)
 640                 return -ENOMEM;
 641         cmd = &fm->cmd.va->fm_exact_match_table;
 642         memset(cmd, 0, sizeof(*cmd));
 643         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 644         cmd->fet_stage = FM_STAGE_LAST;
 645         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 646         if (key == NULL) {
 647                 hdr = &cmd->fet_key.fk_hdrset[0];
 648                 memset(hdr, 0, sizeof(*hdr));
 649                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 650                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 651                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 652                 hdr->l4.udp.fk_source = 0xFFFF;
 653                 hdr->l4.udp.fk_dest = 0xFFFF;
 654                 fet->default_key = 1;
 655         } else {
 656                 memcpy(&cmd->fet_key, key, sizeof(*key));
 657                 memcpy(&fet->key, key, sizeof(*key));
 658                 fet->default_key = 0;
 659         }
 660         cmd->fet_key.fk_packet_tag = 1;
 661
 662         args[0] = FM_EXACT_TABLE_ALLOC;
 663         args[1] = fm->cmd.pa;
 664         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
 665         if (ret) {
 666                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 667                 free(fet);
 668                 return ret;
 669         }
 670         fet->handle = args[0];
 671         fet->ingress = ingress;
 672         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 673                     fet->handle);
 674         *fet_out = fet;
 675         return 0;
 676 }
 677
 678 static void
 679 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 680 {
 681         ENICPMD_FUNC_TRACE();
 682         enic_fm_tbl_free(fm, fet->handle);
 683         if (!fet->default_key)
 684                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 685         free(fet);
 686 }
 687
 688 /*
 689  * Get the exact match table for the given combination of
 690  * <group, ingress, key>. Allocate one on the fly as necessary.
 691  */
 692 static int
 693 enic_fet_get(struct enic_flowman *fm,
 694              uint32_t group,
 695              uint8_t ingress,
 696              struct fm_key_template *key,
 697              struct enic_fm_fet **fet_out,
 698              struct rte_flow_error *error)
 699 {
 700         struct enic_fm_fet *fet;
 701
 702         ENICPMD_FUNC_TRACE();
 703         /* See if we already have this table open */
 704         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 705                 if (fet->group == group && fet->ingress == ingress)
 706                         break;
 707         }
 708         if (fet == NULL) {
 709                 /* Jumping to a non-existing group? Use the default table */
 710                 if (key == NULL) {
 711                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 712                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 713                         return rte_flow_error_set(error, EINVAL,
 714                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 715                                 NULL, "enic: cannot get exact match table");
 716                 }
 717                 fet->group = group;
 718                 /* Default table is never on the open table list */
 719                 if (!fet->default_key)
 720                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 721         }
 722         fet->ref++;
 723         *fet_out = fet;
 724         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 725                     fet->default_key ? "default" : "",
 726                     fet->ingress ? "ingress" : "egress",
 727                     fet->group, fet->ref);
 728         return 0;
 729 }
 730
 731 static void
 732 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 733 {
 734         ENICPMD_FUNC_TRACE();
 735         RTE_ASSERT(fet->ref > 0);
 736         fet->ref--;
 737         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 738                     fet->default_key ? "default" : "",
 739                     fet->ingress ? "ingress" : "egress",
 740                     fet->group, fet->ref);
 741         if (fet->ref == 0)
 742                 enic_fet_free(fm, fet);
 743 }
 744
 745 /* Return 1 if current item is valid on top of the previous one. */
 746 static int
 747 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 748                        const struct enic_fm_items *item_info,
 749                        uint8_t is_first_item)
 750 {
 751         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 752
 753         ENICPMD_FUNC_TRACE();
 754         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 755                 if (prev_item == *allowed_items)
 756                         return 1;
 757         }
 758
 759         /* This is the first item in the stack. Check if that's cool */
 760         if (is_first_item && item_info->valid_start_item)
 761                 return 1;
 762         return 0;
 763 }
 764
 765 /*
 766  * Build the flow manager match entry structure from the provided pattern.
 767  * The pattern is validated as the items are copied.
 768  */
 769 static int
 770 enic_fm_copy_entry(struct enic_flowman *fm,
 771                    const struct rte_flow_item pattern[],
 772                    struct rte_flow_error *error)
 773 {
 774         const struct enic_fm_items *item_info;
 775         enum rte_flow_item_type prev_item;
 776         const struct rte_flow_item *item;
 777         struct copy_item_args args;
 778         uint8_t prev_header_level;
 779         uint8_t is_first_item;
 780         int ret;
 781
 782         ENICPMD_FUNC_TRACE();
 783         item = pattern;
 784         is_first_item = 1;
 785         prev_item = RTE_FLOW_ITEM_TYPE_END;
 786
 787         args.fm_tcam_entry = &fm->tcam_entry;
 788         args.header_level = 0;
 789         prev_header_level = 0;
 790         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 791                 /*
 792                  * Get info about how to validate and copy the item. If NULL
 793                  * is returned the nic does not support the item.
 794                  */
 795                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 796                         continue;
 797
 798                 item_info = &enic_fm_items[item->type];
 799
 800                 if (item->type > FM_MAX_ITEM_TYPE ||
 801                     item_info->copy_item == NULL) {
 802                         return rte_flow_error_set(error, ENOTSUP,
 803                                 RTE_FLOW_ERROR_TYPE_ITEM,
 804                                 NULL, "enic: unsupported item");
 805                 }
 806
 807                 /* check to see if item stacking is valid */
 808                 if (!fm_item_stacking_valid(prev_item, item_info,
 809                                             is_first_item))
 810                         goto stacking_error;
 811
 812                 args.item = item;
 813                 ret = item_info->copy_item(&args);
 814                 if (ret)
 815                         goto item_not_supported;
 816                 /* Going from outer to inner? Treat it as a new packet start */
 817                 if (prev_header_level != args.header_level) {
 818                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 819                         is_first_item = 1;
 820                 } else {
 821                         prev_item = item->type;
 822                         is_first_item = 0;
 823                 }
 824                 prev_header_level = args.header_level;
 825         }
 826         return 0;
 827
 828 item_not_supported:
 829         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 830                                   NULL, "enic: unsupported item type");
 831
 832 stacking_error:
 833         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 834                                   item, "enic: unsupported item stack");
 835 }
 836
 837 static void
 838 flow_item_skip_void(const struct rte_flow_item **item)
 839 {
 840         for ( ; ; (*item)++)
 841                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 842                         return;
 843 }
 844
 845 static void
 846 append_template(void **template, uint8_t *off, const void *data, int len)
 847 {
 848         memcpy(*template, data, len);
 849         *template = (char *)*template + len;
 850         *off = *off + len;
 851 }
 852
 853 static int
 854 enic_fm_append_action_op(struct enic_flowman *fm,
 855                          struct fm_action_op *fm_op,
 856                          struct rte_flow_error *error)
 857 {
 858         int count;
 859
 860         count = fm->action_op_count;
 861         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 862                     count, fm_op->fa_op);
 863         if (count == FM_ACTION_OP_MAX) {
 864                 return rte_flow_error_set(error, EINVAL,
 865                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 866                         "too many action operations");
 867         }
 868         fm->action.fma_action_ops[count] = *fm_op;
 869         fm->action_op_count = count + 1;
 870         return 0;
 871 }
 872
 873 /* NIC requires that 1st steer appear before decap.
 874  * Correct example: steer, decap, steer, steer, ...
 875  */
 876 static void
 877 enic_fm_reorder_action_op(struct enic_flowman *fm)
 878 {
 879         struct fm_action_op *op, *steer, *decap;
 880         struct fm_action_op tmp_op;
 881
 882         ENICPMD_FUNC_TRACE();
 883         /* Find 1st steer and decap */
 884         op = fm->action.fma_action_ops;
 885         steer = NULL;
 886         decap = NULL;
 887         while (op->fa_op != FMOP_END) {
 888                 if (!decap && op->fa_op == FMOP_DECAP_NOSTRIP)
 889                         decap = op;
 890                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
 891                         steer = op;
 892                 op++;
 893         }
 894         /* If decap is before steer, swap */
 895         if (steer && decap && decap < steer) {
 896                 op = fm->action.fma_action_ops;
 897                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
 898                             (long)(decap - op), (long)(steer - op));
 899                 tmp_op = *decap;
 900                 *decap = *steer;
 901                 *steer = tmp_op;
 902         }
 903 }
 904
 905 /* VXLAN decap is done via flowman compound action */
 906 static int
 907 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 908                          struct fm_tcam_match_entry *fmt,
 909                          const struct rte_flow_action *action,
 910                          struct rte_flow_error *error)
 911 {
 912         struct fm_header_set *fm_data;
 913         struct fm_action_op fm_op;
 914
 915         ENICPMD_FUNC_TRACE();
 916         fm_data = &fmt->ftm_data.fk_hdrset[0];
 917         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 918                 return rte_flow_error_set(error, EINVAL,
 919                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 920                         "vxlan-decap: vxlan must be in pattern");
 921         }
 922
 923         memset(&fm_op, 0, sizeof(fm_op));
 924         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
 925         return enic_fm_append_action_op(fm, &fm_op, error);
 926 }
 927
 928 /* VXLAN encap is done via flowman compound action */
 929 static int
 930 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
 931                          const struct rte_flow_item *item,
 932                          struct rte_flow_error *error)
 933 {
 934         struct fm_action_op fm_op;
 935         struct rte_ether_hdr *eth;
 936         uint16_t *ethertype;
 937         void *template;
 938         uint8_t off;
 939
 940         ENICPMD_FUNC_TRACE();
 941         memset(&fm_op, 0, sizeof(fm_op));
 942         fm_op.fa_op = FMOP_ENCAP;
 943         template = fm->action.fma_data;
 944         off = 0;
 945         /*
 946          * Copy flow items to the flowman template starting L2.
 947          * L2 must be ethernet.
 948          */
 949         flow_item_skip_void(&item);
 950         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
 951                 return rte_flow_error_set(error, EINVAL,
 952                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 953                         "vxlan-encap: first item should be ethernet");
 954         eth = (struct rte_ether_hdr *)template;
 955         ethertype = &eth->ether_type;
 956         append_template(&template, &off, item->spec,
 957                         sizeof(struct rte_flow_item_eth));
 958         item++;
 959         flow_item_skip_void(&item);
 960         /* Optional VLAN */
 961         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
 962                 const struct rte_flow_item_vlan *spec;
 963
 964                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
 965                 spec = item->spec;
 966                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
 967                 item++;
 968                 flow_item_skip_void(&item);
 969         }
 970         /* L3 must be IPv4, IPv6 */
 971         switch (item->type) {
 972         case RTE_FLOW_ITEM_TYPE_IPV4:
 973         {
 974                 struct rte_ipv4_hdr *ip4;
 975
 976                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
 977                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 978                 ip4 = (struct rte_ipv4_hdr *)template;
 979                 /*
 980                  * Offset of IPv4 length field and its initial value
 981                  * (IP + UDP + VXLAN) are specified in the action. The NIC
 982                  * will add inner packet length.
 983                  */
 984                 fm_op.encap.len1_offset = off +
 985                         offsetof(struct rte_ipv4_hdr, total_length);
 986                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
 987                         sizeof(struct rte_udp_hdr) +
 988                         sizeof(struct rte_vxlan_hdr);
 989                 append_template(&template, &off, item->spec,
 990                                 sizeof(struct rte_ipv4_hdr));
 991                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
 992                 if (ip4->time_to_live == 0)
 993                         ip4->time_to_live = IP_DEFTTL;
 994                 ip4->next_proto_id = IPPROTO_UDP;
 995                 break;
 996         }
 997         case RTE_FLOW_ITEM_TYPE_IPV6:
 998         {
 999                 struct rte_ipv6_hdr *ip6;
1000
1001                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1002                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1003                 ip6 = (struct rte_ipv6_hdr *)template;
1004                 fm_op.encap.len1_offset = off +
1005                         offsetof(struct rte_ipv6_hdr, payload_len);
1006                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1007                         sizeof(struct rte_vxlan_hdr);
1008                 append_template(&template, &off, item->spec,
1009                                 sizeof(struct rte_ipv6_hdr));
1010                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1011                 if (ip6->hop_limits == 0)
1012                         ip6->hop_limits = IP_DEFTTL;
1013                 ip6->proto = IPPROTO_UDP;
1014                 break;
1015         }
1016         default:
1017                 return rte_flow_error_set(error,
1018                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1019                         "vxlan-encap: L3 must be IPv4/IPv6");
1020         }
1021         item++;
1022         flow_item_skip_void(&item);
1023
1024         /* L4 is UDP */
1025         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1026                 return rte_flow_error_set(error, EINVAL,
1027                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1028                         "vxlan-encap: UDP must follow IPv4/IPv6");
1029         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1030         fm_op.encap.len2_offset =
1031                 off + offsetof(struct rte_udp_hdr, dgram_len);
1032         fm_op.encap.len2_delta =
1033                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1034         append_template(&template, &off, item->spec,
1035                         sizeof(struct rte_udp_hdr));
1036         item++;
1037         flow_item_skip_void(&item);
1038
1039         /* Finally VXLAN */
1040         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1041                 return rte_flow_error_set(error,
1042                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1043                         "vxlan-encap: VXLAN must follow UDP");
1044         append_template(&template, &off, item->spec,
1045                         sizeof(struct rte_flow_item_vxlan));
1046
1047         /*
1048          * Fill in the rest of the action structure.
1049          * Indicate that we want to encap with vxlan at packet start.
1050          */
1051         fm_op.encap.template_offset = 0;
1052         fm_op.encap.template_len = off;
1053         return enic_fm_append_action_op(fm, &fm_op, error);
1054 }
1055
1056 static int
1057 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1058                   uint64_t *handle)
1059 {
1060         uint32_t bdf;
1061         uint64_t args[2];
1062         int rc;
1063
1064         ENICPMD_FUNC_TRACE();
1065         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1066                     addr->function);
1067         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1068         args[0] = FM_VNIC_FIND;
1069         args[1] = bdf;
1070         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1071         if (rc != 0) {
1072                 ENICPMD_LOG(ERR, "allocating counters rc=%d", rc);
1073                 return rc;
1074         }
1075         *handle = args[0];
1076         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1077         return 0;
1078 }
1079
1080 /* Translate flow actions to flowman TCAM entry actions */
1081 static int
1082 enic_fm_copy_action(struct enic_flowman *fm,
1083                     const struct rte_flow_action actions[],
1084                     uint8_t ingress,
1085                     struct rte_flow_error *error)
1086 {
1087         enum {
1088                 FATE = 1 << 0,
1089                 MARK = 1 << 1,
1090                 PASSTHRU = 1 << 2,
1091                 COUNT = 1 << 3,
1092                 ENCAP = 1 << 4,
1093                 DECAP = 1 << 5,
1094         };
1095         struct fm_tcam_match_entry *fmt;
1096         struct fm_action_op fm_op;
1097         struct enic *enic;
1098         uint32_t overlap;
1099         uint64_t vnic_h;
1100         bool first_rq;
1101         int ret;
1102
1103         ENICPMD_FUNC_TRACE();
1104         fmt = &fm->tcam_entry;
1105         first_rq = true;
1106         enic = fm->enic;
1107         overlap = 0;
1108         vnic_h = 0; /* 0 = current vNIC */
1109         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1110                 switch (actions->type) {
1111                 case RTE_FLOW_ACTION_TYPE_VOID:
1112                         continue;
1113                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1114                         if (overlap & PASSTHRU)
1115                                 goto unsupported;
1116                         overlap |= PASSTHRU;
1117                         break;
1118                 }
1119                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1120                         const struct rte_flow_action_jump *jump =
1121                                 actions->conf;
1122                         struct enic_fm_fet *fet;
1123
1124                         if (overlap & FATE)
1125                                 goto unsupported;
1126                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1127                                            &fet, error);
1128                         if (ret)
1129                                 return ret;
1130                         overlap |= FATE;
1131                         memset(&fm_op, 0, sizeof(fm_op));
1132                         fm_op.fa_op = FMOP_EXACT_MATCH;
1133                         fm_op.exact.handle = fet->handle;
1134                         fm->fet = fet;
1135                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1136                         if (ret)
1137                                 return ret;
1138                         break;
1139                 }
1140                 case RTE_FLOW_ACTION_TYPE_MARK: {
1141                         const struct rte_flow_action_mark *mark =
1142                                 actions->conf;
1143
1144                         if (overlap & MARK)
1145                                 goto unsupported;
1146                         overlap |= MARK;
1147                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1148                                 return rte_flow_error_set(error, EINVAL,
1149                                         RTE_FLOW_ERROR_TYPE_ACTION,
1150                                         NULL, "invalid mark id");
1151                         memset(&fm_op, 0, sizeof(fm_op));
1152                         fm_op.fa_op = FMOP_MARK;
1153                         fm_op.mark.mark = mark->id + 1;
1154                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1155                         if (ret)
1156                                 return ret;
1157                         break;
1158                 }
1159                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1160                         if (overlap & MARK)
1161                                 goto unsupported;
1162                         overlap |= MARK;
1163                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1164                         memset(&fm_op, 0, sizeof(fm_op));
1165                         fm_op.fa_op = FMOP_MARK;
1166                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1167                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1168                         if (ret)
1169                                 return ret;
1170                         break;
1171                 }
1172                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1173                         const struct rte_flow_action_queue *queue =
1174                                 actions->conf;
1175
1176                         /*
1177                          * If fate other than QUEUE or RSS, fail. Multiple
1178                          * rss and queue actions are ok.
1179                          */
1180                         if ((overlap & FATE) && first_rq)
1181                                 goto unsupported;
1182                         first_rq = false;
1183                         overlap |= FATE;
1184                         memset(&fm_op, 0, sizeof(fm_op));
1185                         fm_op.fa_op = FMOP_RQ_STEER;
1186                         fm_op.rq_steer.rq_index =
1187                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1188                         fm_op.rq_steer.rq_count = 1;
1189                         fm_op.rq_steer.vnic_handle = vnic_h;
1190                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1191                         if (ret)
1192                                 return ret;
1193                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1194                                     fm_op.rq_steer.rq_index);
1195                         break;
1196                 }
1197                 case RTE_FLOW_ACTION_TYPE_DROP: {
1198                         if (overlap & FATE)
1199                                 goto unsupported;
1200                         overlap |= FATE;
1201                         memset(&fm_op, 0, sizeof(fm_op));
1202                         fm_op.fa_op = FMOP_DROP;
1203                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1204                         if (ret)
1205                                 return ret;
1206                         ENICPMD_LOG(DEBUG, "create DROP action");
1207                         break;
1208                 }
1209                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1210                         if (overlap & COUNT)
1211                                 goto unsupported;
1212                         overlap |= COUNT;
1213                         /* Count is associated with entry not action on VIC. */
1214                         fmt->ftm_flags |= FMEF_COUNTER;
1215                         break;
1216                 }
1217                 case RTE_FLOW_ACTION_TYPE_RSS: {
1218                         const struct rte_flow_action_rss *rss = actions->conf;
1219                         bool allow;
1220                         uint16_t i;
1221
1222                         /*
1223                          * If fate other than QUEUE or RSS, fail. Multiple
1224                          * rss and queue actions are ok.
1225                          */
1226                         if ((overlap & FATE) && first_rq)
1227                                 goto unsupported;
1228                         first_rq = false;
1229                         overlap |= FATE;
1230
1231                         /*
1232                          * Hardware only supports RSS actions on outer level
1233                          * with default type and function. Queues must be
1234                          * sequential.
1235                          */
1236                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1237                                 rss->level == 0 && (rss->types == 0 ||
1238                                 rss->types == enic->rss_hf) &&
1239                                 rss->queue_num <= enic->rq_count &&
1240                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1241
1242
1243                         /* Identity queue map needs to be sequential */
1244                         for (i = 1; i < rss->queue_num; i++)
1245                                 allow = allow && (rss->queue[i] ==
1246                                         rss->queue[i - 1] + 1);
1247                         if (!allow)
1248                                 goto unsupported;
1249
1250                         memset(&fm_op, 0, sizeof(fm_op));
1251                         fm_op.fa_op = FMOP_RQ_STEER;
1252                         fm_op.rq_steer.rq_index =
1253                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1254                         fm_op.rq_steer.rq_count = rss->queue_num;
1255                         fm_op.rq_steer.vnic_handle = vnic_h;
1256                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1257                         if (ret)
1258                                 return ret;
1259                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1260                                     fm_op.rq_steer.rq_index);
1261                         break;
1262                 }
1263                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1264                         const struct rte_flow_action_port_id *port;
1265                         struct rte_pci_device *pdev;
1266                         struct rte_eth_dev *dev;
1267
1268                         port = actions->conf;
1269                         if (port->original) {
1270                                 vnic_h = 0; /* This port */
1271                                 break;
1272                         }
1273                         ENICPMD_LOG(DEBUG, "port id %u", port->id);
1274                         if (!rte_eth_dev_is_valid_port(port->id)) {
1275                                 return rte_flow_error_set(error, EINVAL,
1276                                         RTE_FLOW_ERROR_TYPE_ACTION,
1277                                         NULL, "invalid port_id");
1278                         }
1279                         dev = &rte_eth_devices[port->id];
1280                         if (!dev_is_enic(dev)) {
1281                                 return rte_flow_error_set(error, EINVAL,
1282                                         RTE_FLOW_ERROR_TYPE_ACTION,
1283                                         NULL, "port_id is not enic");
1284                         }
1285                         pdev = RTE_ETH_DEV_TO_PCI(dev);
1286                         if (enic_fm_find_vnic(enic, &pdev->addr, &vnic_h)) {
1287                                 return rte_flow_error_set(error, EINVAL,
1288                                         RTE_FLOW_ERROR_TYPE_ACTION,
1289                                         NULL, "port_id is not vnic");
1290                         }
1291                         break;
1292                 }
1293                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1294                         if (overlap & DECAP)
1295                                 goto unsupported;
1296                         overlap |= DECAP;
1297
1298                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1299                                 error);
1300                         if (ret != 0)
1301                                 return ret;
1302                         break;
1303                 }
1304                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1305                         const struct rte_flow_action_vxlan_encap *encap;
1306
1307                         encap = actions->conf;
1308                         if (overlap & ENCAP)
1309                                 goto unsupported;
1310                         overlap |= ENCAP;
1311                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1312                                 error);
1313                         if (ret != 0)
1314                                 return ret;
1315                         break;
1316                 }
1317                 default:
1318                         goto unsupported;
1319                 }
1320         }
1321
1322         if (!(overlap & (FATE | PASSTHRU | COUNT)))
1323                 goto unsupported;
1324         memset(&fm_op, 0, sizeof(fm_op));
1325         fm_op.fa_op = FMOP_END;
1326         ret = enic_fm_append_action_op(fm, &fm_op, error);
1327         if (ret)
1328                 return ret;
1329         enic_fm_reorder_action_op(fm);
1330         return 0;
1331
1332 unsupported:
1333         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1334                                   NULL, "enic: unsupported action");
1335 }
1336
1337 /** Check if the action is supported */
1338 static int
1339 enic_fm_match_action(const struct rte_flow_action *action,
1340                      const enum rte_flow_action_type *supported_actions)
1341 {
1342         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1343              supported_actions++) {
1344                 if (action->type == *supported_actions)
1345                         return 1;
1346         }
1347         return 0;
1348 }
1349
1350 /* Debug function to dump internal NIC action structure. */
1351 static void
1352 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1353 {
1354         /* Manually keep in sync with FMOP commands */
1355         const char *fmop_str[FMOP_OP_MAX] = {
1356                 [FMOP_END] = "end",
1357                 [FMOP_DROP] = "drop",
1358                 [FMOP_RQ_STEER] = "steer",
1359                 [FMOP_EXACT_MATCH] = "exmatch",
1360                 [FMOP_MARK] = "mark",
1361                 [FMOP_EXT_MARK] = "ext_mark",
1362                 [FMOP_TAG] = "tag",
1363                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1364                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1365                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1366                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1367                 [FMOP_ENCAP] = "encap",
1368                 [FMOP_SET_OVLAN] = "set_ovlan",
1369                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1370         };
1371         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1372         char buf[128], *bp = buf;
1373         const char *op_str;
1374         int i, n, buf_len;
1375
1376         buf[0] = '\0';
1377         buf_len = sizeof(buf);
1378         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1379                 if (op->fa_op == FMOP_END)
1380                         break;
1381                 if (op->fa_op >= FMOP_OP_MAX)
1382                         op_str = "unknown";
1383                 else
1384                         op_str = fmop_str[op->fa_op];
1385                 n = snprintf(bp, buf_len, "%s,", op_str);
1386                 if (n > 0 && n < buf_len) {
1387                         bp += n;
1388                         buf_len -= n;
1389                 }
1390                 op++;
1391         }
1392         /* Remove trailing comma */
1393         if (buf[0])
1394                 *(bp - 1) = '\0';
1395         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1396 }
1397
1398 static int
1399 bits_to_str(uint32_t bits, const char *strings[], int max,
1400             char *buf, int buf_len)
1401 {
1402         int i, n = 0, len = 0;
1403
1404         for (i = 0; i < max; i++) {
1405                 if (bits & (1 << i)) {
1406                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1407                         if (n > 0 && n < buf_len) {
1408                                 buf += n;
1409                                 buf_len -= n;
1410                                 len += n;
1411                         }
1412                 }
1413         }
1414         /* Remove trailing comma */
1415         if (len) {
1416                 *(buf - 1) = '\0';
1417                 len--;
1418         }
1419         return len;
1420 }
1421
1422 /* Debug function to dump internal NIC filter structure. */
1423 static void
1424 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1425                           int buf_len)
1426 {
1427         /* Manually keep in sync with FKM_BITS */
1428         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1429                 [FKM_QTAG_BIT] = "qtag",
1430                 [FKM_CMD_BIT] = "cmd",
1431                 [FKM_IPV4_BIT] = "ip4",
1432                 [FKM_IPV6_BIT] = "ip6",
1433                 [FKM_ROCE_BIT] = "roce",
1434                 [FKM_UDP_BIT] = "udp",
1435                 [FKM_TCP_BIT] = "tcp",
1436                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1437                 [FKM_IPFRAG_BIT] = "ipfrag",
1438                 [FKM_NVGRE_BIT] = "nvgre",
1439                 [FKM_VXLAN_BIT] = "vxlan",
1440                 [FKM_GENEVE_BIT] = "geneve",
1441                 [FKM_NSH_BIT] = "nsh",
1442                 [FKM_ROCEV2_BIT] = "rocev2",
1443                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1444                 [FKM_IPOK_BIT] = "ipok",
1445                 [FKM_L4OK_BIT] = "l4ok",
1446                 [FKM_ROCEOK_BIT] = "roceok",
1447                 [FKM_FCSOK_BIT] = "fcsok",
1448                 [FKM_EG_SPAN_BIT] = "eg_span",
1449                 [FKM_IG_SPAN_BIT] = "ig_span",
1450                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1451         };
1452         /* Manually keep in sync with FKH_BITS */
1453         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1454                 [FKH_ETHER_BIT] = "eth",
1455                 [FKH_QTAG_BIT] = "qtag",
1456                 [FKH_L2RAW_BIT] = "l2raw",
1457                 [FKH_IPV4_BIT] = "ip4",
1458                 [FKH_IPV6_BIT] = "ip6",
1459                 [FKH_L3RAW_BIT] = "l3raw",
1460                 [FKH_UDP_BIT] = "udp",
1461                 [FKH_TCP_BIT] = "tcp",
1462                 [FKH_ICMP_BIT] = "icmp",
1463                 [FKH_VXLAN_BIT] = "vxlan",
1464                 [FKH_L4RAW_BIT] = "l4raw",
1465         };
1466         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1467         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1468         int n;
1469
1470         if (!fkm_bits && !fkh_bits)
1471                 return;
1472         n = snprintf(buf, buf_len, "metadata(");
1473         if (n > 0 && n < buf_len) {
1474                 buf += n;
1475                 buf_len -= n;
1476         }
1477         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1478         if (n > 0 && n < buf_len) {
1479                 buf += n;
1480                 buf_len -= n;
1481         }
1482         n = snprintf(buf, buf_len, ") valid hdr fields(");
1483         if (n > 0 && n < buf_len) {
1484                 buf += n;
1485                 buf_len -= n;
1486         }
1487         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1488         if (n > 0 && n < buf_len) {
1489                 buf += n;
1490                 buf_len -= n;
1491         }
1492         snprintf(buf, buf_len, ")");
1493 }
1494
1495 static void
1496 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1497                         uint8_t ingress)
1498 {
1499         char buf[256];
1500
1501         memset(buf, 0, sizeof(buf));
1502         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1503                                   buf, sizeof(buf));
1504         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1505                     (ingress) ? "IG" : "EG", buf,
1506                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1507         memset(buf, 0, sizeof(buf));
1508         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1509                                   buf, sizeof(buf));
1510         if (buf[0])
1511                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1512 }
1513
1514 /* Debug function to dump internal NIC flow structures. */
1515 static void
1516 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1517                         const struct fm_action *fm_action,
1518                         uint8_t ingress)
1519 {
1520         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1521                 return;
1522         enic_fm_dump_tcam_match(fm_match, ingress);
1523         enic_fm_dump_tcam_actions(fm_action);
1524 }
1525
1526 static int
1527 enic_fm_flow_parse(struct enic_flowman *fm,
1528                    const struct rte_flow_attr *attrs,
1529                    const struct rte_flow_item pattern[],
1530                    const struct rte_flow_action actions[],
1531                    struct rte_flow_error *error)
1532 {
1533         const struct rte_flow_action *action;
1534         unsigned int ret;
1535         static const enum rte_flow_action_type *sa;
1536
1537         ENICPMD_FUNC_TRACE();
1538         ret = 0;
1539         if (!pattern) {
1540                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1541                                    NULL, "no pattern specified");
1542                 return -rte_errno;
1543         }
1544
1545         if (!actions) {
1546                 rte_flow_error_set(error, EINVAL,
1547                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1548                                    NULL, "no action specified");
1549                 return -rte_errno;
1550         }
1551
1552         if (attrs) {
1553                 if (attrs->priority) {
1554                         rte_flow_error_set(error, ENOTSUP,
1555                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1556                                            NULL,
1557                                            "priorities are not supported");
1558                         return -rte_errno;
1559                 } else if (attrs->transfer) {
1560                         rte_flow_error_set(error, ENOTSUP,
1561                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1562                                            NULL,
1563                                            "transfer is not supported");
1564                         return -rte_errno;
1565                 } else if (attrs->ingress && attrs->egress) {
1566                         rte_flow_error_set(error, ENOTSUP,
1567                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1568                                            NULL,
1569                                            "bidirectional rules not supported");
1570                         return -rte_errno;
1571                 }
1572
1573         } else {
1574                 rte_flow_error_set(error, EINVAL,
1575                                    RTE_FLOW_ERROR_TYPE_ATTR,
1576                                    NULL, "no attribute specified");
1577                 return -rte_errno;
1578         }
1579
1580         /* Verify Actions. */
1581         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1582              enic_fm_supported_eg_actions;
1583         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1584              action++) {
1585                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1586                         continue;
1587                 else if (!enic_fm_match_action(action, sa))
1588                         break;
1589         }
1590         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1591                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1592                                    action, "invalid action");
1593                 return -rte_errno;
1594         }
1595         ret = enic_fm_copy_entry(fm, pattern, error);
1596         if (ret)
1597                 return ret;
1598         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1599         return ret;
1600 }
1601
1602 static void
1603 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1604 {
1605         if (!fm_flow->counter_valid)
1606                 return;
1607         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1608         fm_flow->counter_valid = false;
1609 }
1610
1611 static int
1612 enic_fm_more_counters(struct enic_flowman *fm)
1613 {
1614         struct enic_fm_counter *new_stack;
1615         struct enic_fm_counter *ctrs;
1616         struct enic *enic;
1617         int i, rc;
1618         uint64_t args[2];
1619
1620         ENICPMD_FUNC_TRACE();
1621         enic = fm->enic;
1622         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1623                                 FM_COUNTERS_EXPAND) *
1624                                 sizeof(struct enic_fm_counter), 0);
1625         if (new_stack == NULL) {
1626                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1627                 return -ENOMEM;
1628         }
1629         fm->counter_stack = new_stack;
1630
1631         args[0] = FM_COUNTER_BRK;
1632         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1633         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1634         if (rc != 0) {
1635                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1636                 return rc;
1637         }
1638         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1639                 fm->counters_alloced;
1640         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1641                 ctrs->handle = fm->counters_alloced + i;
1642                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1643         }
1644         fm->counters_alloced += FM_COUNTERS_EXPAND;
1645         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1646                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1647         return 0;
1648 }
1649
1650 static int
1651 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1652 {
1653         struct enic *enic;
1654         uint64_t args[3];
1655         int ret;
1656
1657         ENICPMD_FUNC_TRACE();
1658         enic = fm->enic;
1659         args[0] = FM_COUNTER_QUERY;
1660         args[1] = c->handle;
1661         args[2] = 1; /* clear */
1662         ret = vnic_dev_flowman_cmd(enic->vdev, args, 3);
1663         if (ret) {
1664                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1665                             ret, c->handle);
1666                 return ret;
1667         }
1668         return 0;
1669 }
1670
1671 static int
1672 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1673                       struct enic_fm_counter **ctr)
1674 {
1675         struct enic_fm_counter *c;
1676         int ret;
1677
1678         ENICPMD_FUNC_TRACE();
1679         *ctr = NULL;
1680         if (SLIST_EMPTY(&fm->counters)) {
1681                 ret = enic_fm_more_counters(fm);
1682                 if (ret)
1683                         return rte_flow_error_set(error, -ret,
1684                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1685                                 NULL, "enic: out of counters");
1686         }
1687         c = SLIST_FIRST(&fm->counters);
1688         SLIST_REMOVE_HEAD(&fm->counters, next);
1689         *ctr = c;
1690         return 0;
1691 }
1692
1693 static int
1694 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1695 {
1696         uint64_t args[2];
1697         int rc;
1698
1699         ENICPMD_FUNC_TRACE();
1700         args[0] = FM_ACTION_FREE;
1701         args[1] = handle;
1702         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1703         if (rc)
1704                 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1705                             rc, handle);
1706         return rc;
1707 }
1708
1709 static int
1710 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1711 {
1712         uint64_t args[2];
1713         int rc;
1714
1715         ENICPMD_FUNC_TRACE();
1716         args[0] = FM_MATCH_ENTRY_REMOVE;
1717         args[1] = handle;
1718         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1719         if (rc)
1720                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1721                             " handle=0x%" PRIx64, rc, handle);
1722         return rc;
1723 }
1724
1725 static struct enic_fm_jump_flow *
1726 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1727 {
1728         struct enic_fm_jump_flow *j;
1729
1730         ENICPMD_FUNC_TRACE();
1731         TAILQ_FOREACH(j, &fm->jump_list, list) {
1732                 if (j->group == group)
1733                         return j;
1734         }
1735         return NULL;
1736 }
1737
1738 static void
1739 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1740 {
1741         struct enic_fm_jump_flow *j;
1742
1743         ENICPMD_FUNC_TRACE();
1744         TAILQ_FOREACH(j, &fm->jump_list, list) {
1745                 if (j->flow == flow) {
1746                         TAILQ_REMOVE(&fm->jump_list, j, list);
1747                         free(j);
1748                         return;
1749                 }
1750         }
1751 }
1752
1753 static int
1754 save_jump_flow(struct enic_flowman *fm,
1755                struct rte_flow *flow,
1756                uint32_t group,
1757                struct fm_tcam_match_entry *match,
1758                struct fm_action *action)
1759 {
1760         struct enic_fm_jump_flow *j;
1761
1762         ENICPMD_FUNC_TRACE();
1763         j = calloc(1, sizeof(struct enic_fm_jump_flow));
1764         if (j == NULL)
1765                 return -ENOMEM;
1766         j->flow = flow;
1767         j->group = group;
1768         j->match = *match;
1769         j->action = *action;
1770         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1771         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1772         return 0;
1773 }
1774
1775 static void
1776 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1777 {
1778         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1779                 enic_fm_entry_free(fm, fm_flow->entry_handle);
1780                 fm_flow->entry_handle = FM_INVALID_HANDLE;
1781         }
1782         if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1783                 enic_fm_action_free(fm, fm_flow->action_handle);
1784                 fm_flow->action_handle = FM_INVALID_HANDLE;
1785         }
1786         enic_fm_counter_free(fm, fm_flow);
1787         if (fm_flow->fet) {
1788                 enic_fet_put(fm, fm_flow->fet);
1789                 fm_flow->fet = NULL;
1790         }
1791 }
1792
1793 static void
1794 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1795 {
1796         if (flow->fm->fet && flow->fm->fet->default_key)
1797                 remove_jump_flow(fm, flow);
1798         __enic_fm_flow_free(fm, flow->fm);
1799         free(flow->fm);
1800         free(flow);
1801 }
1802
1803 static int
1804 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1805                        struct fm_tcam_match_entry *match_in,
1806                        uint64_t *entry_handle,
1807                        uint8_t ingress,
1808                        struct rte_flow_error *error)
1809 {
1810         struct fm_tcam_match_entry *ftm;
1811         uint64_t args[3];
1812         int ret;
1813
1814         ENICPMD_FUNC_TRACE();
1815         /* Copy entry to the command buffer */
1816         ftm = &fm->cmd.va->fm_tcam_match_entry;
1817         memcpy(ftm, match_in, sizeof(*ftm));
1818         /* Add TCAM entry */
1819         args[0] = FM_TCAM_ENTRY_INSTALL;
1820         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1821         args[2] = fm->cmd.pa;
1822         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1823         if (ret != 0) {
1824                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1825                             ingress ? "ingress" : "egress", ret);
1826                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1827                         NULL, "enic: devcmd(tcam-entry-install)");
1828                 return ret;
1829         }
1830         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1831                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
1832         *entry_handle = args[0];
1833         return 0;
1834 }
1835
1836 static int
1837 enic_fm_add_exact_entry(struct enic_flowman *fm,
1838                         struct fm_tcam_match_entry *match_in,
1839                         uint64_t *entry_handle,
1840                         struct enic_fm_fet *fet,
1841                         struct rte_flow_error *error)
1842 {
1843         struct fm_exact_match_entry *fem;
1844         uint64_t args[3];
1845         int ret;
1846
1847         ENICPMD_FUNC_TRACE();
1848         /* The new entry must have the table's key */
1849         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1850                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1851                 return rte_flow_error_set(error, EINVAL,
1852                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1853                         "enic: key does not match group's key");
1854         }
1855
1856         /* Copy entry to the command buffer */
1857         fem = &fm->cmd.va->fm_exact_match_entry;
1858         /*
1859          * Translate TCAM entry to exact entry. As is only need to drop
1860          * position and mask. The mask is part of the exact match table.
1861          * Position (aka priority) is not supported in the exact match table.
1862          */
1863         fem->fem_data = match_in->ftm_data;
1864         fem->fem_flags = match_in->ftm_flags;
1865         fem->fem_action = match_in->ftm_action;
1866         fem->fem_counter = match_in->ftm_counter;
1867
1868         /* Add exact entry */
1869         args[0] = FM_EXACT_ENTRY_INSTALL;
1870         args[1] = fet->handle;
1871         args[2] = fm->cmd.pa;
1872         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1873         if (ret != 0) {
1874                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
1875                             fet->ingress ? "ingress" : "egress", fet->group);
1876                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1877                         NULL, "enic: devcmd(exact-entry-install)");
1878                 return ret;
1879         }
1880         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
1881                     " handle=0x%" PRIx64,
1882                     fet->ingress ? "ingress" : "egress", fet->group,
1883                     (uint64_t)args[0]);
1884         *entry_handle = args[0];
1885         return 0;
1886 }
1887
1888 /* Push match-action to the NIC. */
1889 static int
1890 __enic_fm_flow_add_entry(struct enic_flowman *fm,
1891                          struct enic_fm_flow *fm_flow,
1892                          struct fm_tcam_match_entry *match_in,
1893                          struct fm_action *action_in,
1894                          uint32_t group,
1895                          uint8_t ingress,
1896                          struct rte_flow_error *error)
1897 {
1898         struct enic_fm_counter *ctr;
1899         struct fm_action *fma;
1900         uint64_t action_h;
1901         uint64_t entry_h;
1902         uint64_t args[3];
1903         int ret;
1904
1905         ENICPMD_FUNC_TRACE();
1906         /* Allocate action. */
1907         fma = &fm->cmd.va->fm_action;
1908         memcpy(fma, action_in, sizeof(*fma));
1909         args[0] = FM_ACTION_ALLOC;
1910         args[1] = fm->cmd.pa;
1911         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1912         if (ret != 0) {
1913                 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
1914                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1915                         NULL, "enic: devcmd(action-alloc)");
1916                 return ret;
1917         }
1918         action_h = args[0];
1919         fm_flow->action_handle = action_h;
1920         match_in->ftm_action = action_h;
1921         ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
1922
1923         /* Allocate counter if requested. */
1924         if (match_in->ftm_flags & FMEF_COUNTER) {
1925                 ret = enic_fm_counter_alloc(fm, error, &ctr);
1926                 if (ret) /* error has been filled in */
1927                         return ret;
1928                 fm_flow->counter_valid = true;
1929                 fm_flow->counter = ctr;
1930                 match_in->ftm_counter = ctr->handle;
1931         }
1932
1933         /*
1934          * Get the group's table (either TCAM or exact match table) and
1935          * add entry to it. If we use the exact match table, the handler
1936          * will translate the TCAM entry (match_in) to the appropriate
1937          * exact match entry and use that instead.
1938          */
1939         entry_h = FM_INVALID_HANDLE;
1940         if (group == FM_TCAM_RTE_GROUP) {
1941                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
1942                                              error);
1943                 if (ret)
1944                         return ret;
1945                 /* Jump action might have a ref to fet */
1946                 fm_flow->fet = fm->fet;
1947                 fm->fet = NULL;
1948         } else {
1949                 struct enic_fm_fet *fet = NULL;
1950
1951                 ret = enic_fet_get(fm, group, ingress,
1952                                    &match_in->ftm_mask, &fet, error);
1953                 if (ret)
1954                         return ret;
1955                 fm_flow->fet = fet;
1956                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
1957                                               error);
1958                 if (ret)
1959                         return ret;
1960         }
1961         /* Clear counter after adding entry, as it requires in-use counter */
1962         if (fm_flow->counter_valid) {
1963                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
1964                 if (ret)
1965                         return ret;
1966         }
1967         fm_flow->entry_handle = entry_h;
1968         return 0;
1969 }
1970
1971 /* Push match-action to the NIC. */
1972 static struct rte_flow *
1973 enic_fm_flow_add_entry(struct enic_flowman *fm,
1974                        struct fm_tcam_match_entry *match_in,
1975                        struct fm_action *action_in,
1976                        const struct rte_flow_attr *attrs,
1977                        struct rte_flow_error *error)
1978 {
1979         struct enic_fm_flow *fm_flow;
1980         struct rte_flow *flow;
1981
1982         ENICPMD_FUNC_TRACE();
1983         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
1984         flow = calloc(1, sizeof(*flow));
1985         fm_flow = calloc(1, sizeof(*fm_flow));
1986         if (flow == NULL || fm_flow == NULL) {
1987                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1988                         NULL, "enic: cannot allocate rte_flow");
1989                 free(flow);
1990                 free(fm_flow);
1991                 return NULL;
1992         }
1993         flow->fm = fm_flow;
1994         fm_flow->action_handle = FM_INVALID_HANDLE;
1995         fm_flow->entry_handle = FM_INVALID_HANDLE;
1996         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
1997                                      attrs->group, attrs->ingress, error)) {
1998                 enic_fm_flow_free(fm, flow);
1999                 return NULL;
2000         }
2001         return flow;
2002 }
2003
2004 static void
2005 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2006                    struct rte_flow_error *error)
2007 {
2008         struct enic_fm_flow *fm_flow;
2009         struct enic_fm_jump_flow *j;
2010         struct fm_action *fma;
2011         uint32_t group;
2012
2013         ENICPMD_FUNC_TRACE();
2014         /*
2015          * Find the saved flows that should jump to the new table (fet).
2016          * Then delete the old TCAM entry that jumps to the default table,
2017          * and add a new one that jumps to the new table.
2018          */
2019         group = fet->group;
2020         j = find_jump_flow(fm, group);
2021         while (j) {
2022                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2023                             j->flow, group);
2024                 /* Delete old entry */
2025                 fm_flow = j->flow->fm;
2026                 __enic_fm_flow_free(fm, fm_flow);
2027
2028                 /* Add new entry */
2029                 fma = &j->action;
2030                 fma->fma_action_ops[0].exact.handle = fet->handle;
2031                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2032                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2033                         /* Cannot roll back changes at the moment */
2034                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2035                                     j->flow);
2036                 } else {
2037                         fm_flow->fet = fet;
2038                         fet->ref++;
2039                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2040                                     fet->group, fet->ref);
2041                 }
2042
2043                 TAILQ_REMOVE(&fm->jump_list, j, list);
2044                 free(j);
2045                 j = find_jump_flow(fm, group);
2046         }
2047 }
2048
2049 static void
2050 enic_fm_open_scratch(struct enic_flowman *fm)
2051 {
2052         fm->action_op_count = 0;
2053         fm->fet = NULL;
2054         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2055         memset(&fm->action, 0, sizeof(fm->action));
2056 }
2057
2058 static void
2059 enic_fm_close_scratch(struct enic_flowman *fm)
2060 {
2061         if (fm->fet) {
2062                 enic_fet_put(fm, fm->fet);
2063                 fm->fet = NULL;
2064         }
2065         fm->action_op_count = 0;
2066 }
2067
2068 static int
2069 enic_fm_flow_validate(struct rte_eth_dev *dev,
2070                       const struct rte_flow_attr *attrs,
2071                       const struct rte_flow_item pattern[],
2072                       const struct rte_flow_action actions[],
2073                       struct rte_flow_error *error)
2074 {
2075         struct fm_tcam_match_entry *fm_tcam_entry;
2076         struct fm_action *fm_action;
2077         struct enic_flowman *fm;
2078         int ret;
2079
2080         ENICPMD_FUNC_TRACE();
2081         fm = pmd_priv(dev)->fm;
2082         if (fm == NULL)
2083                 return -ENOTSUP;
2084         enic_fm_open_scratch(fm);
2085         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2086         if (!ret) {
2087                 fm_tcam_entry = &fm->tcam_entry;
2088                 fm_action = &fm->action;
2089                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2090                                         attrs->ingress);
2091         }
2092         enic_fm_close_scratch(fm);
2093         return ret;
2094 }
2095
2096 static int
2097 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2098                          struct rte_flow *flow, void *data,
2099                          struct rte_flow_error *error)
2100 {
2101         struct rte_flow_query_count *query;
2102         struct enic_fm_flow *fm_flow;
2103         struct enic *enic;
2104         uint64_t args[3];
2105         int rc;
2106
2107         ENICPMD_FUNC_TRACE();
2108         enic = pmd_priv(dev);
2109         query = data;
2110         fm_flow = flow->fm;
2111         if (!fm_flow->counter_valid)
2112                 return rte_flow_error_set(error, ENOTSUP,
2113                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2114                         "enic: flow does not have counter");
2115
2116         args[0] = FM_COUNTER_QUERY;
2117         args[1] = fm_flow->counter->handle;
2118         args[2] = query->reset;
2119         rc = vnic_dev_flowman_cmd(enic->vdev, args, 3);
2120         if (rc) {
2121                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2122                             rc, fm_flow->counter->handle);
2123                 return rc;
2124         }
2125         query->hits_set = 1;
2126         query->hits = args[0];
2127         query->bytes_set = 1;
2128         query->bytes = args[1];
2129         return 0;
2130 }
2131
2132 static int
2133 enic_fm_flow_query(struct rte_eth_dev *dev,
2134                    struct rte_flow *flow,
2135                    const struct rte_flow_action *actions,
2136                    void *data,
2137                    struct rte_flow_error *error)
2138 {
2139         int ret = 0;
2140
2141         ENICPMD_FUNC_TRACE();
2142         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2143                 switch (actions->type) {
2144                 case RTE_FLOW_ACTION_TYPE_VOID:
2145                         break;
2146                 case RTE_FLOW_ACTION_TYPE_COUNT:
2147                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2148                         break;
2149                 default:
2150                         return rte_flow_error_set(error, ENOTSUP,
2151                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2152                                                   actions,
2153                                                   "action not supported");
2154                 }
2155                 if (ret < 0)
2156                         return ret;
2157         }
2158         return 0;
2159 }
2160
2161 static struct rte_flow *
2162 enic_fm_flow_create(struct rte_eth_dev *dev,
2163                     const struct rte_flow_attr *attrs,
2164                     const struct rte_flow_item pattern[],
2165                     const struct rte_flow_action actions[],
2166                     struct rte_flow_error *error)
2167 {
2168         struct fm_tcam_match_entry *fm_tcam_entry;
2169         struct fm_action *fm_action;
2170         struct enic_flowman *fm;
2171         struct enic_fm_fet *fet;
2172         struct rte_flow *flow;
2173         struct enic *enic;
2174         int ret;
2175
2176         ENICPMD_FUNC_TRACE();
2177         enic = pmd_priv(dev);
2178         fm = enic->fm;
2179         if (fm == NULL) {
2180                 rte_flow_error_set(error, ENOTSUP,
2181                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2182                         "flowman is not initialized");
2183                 return NULL;
2184         }
2185         enic_fm_open_scratch(fm);
2186         flow = NULL;
2187         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2188         if (ret < 0)
2189                 goto error_with_scratch;
2190         fm_tcam_entry = &fm->tcam_entry;
2191         fm_action = &fm->action;
2192         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2193                                       attrs, error);
2194         if (flow) {
2195                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2196                 fet = flow->fm->fet;
2197                 if (fet && fet->default_key) {
2198                         /*
2199                          * Jump to non-existent group? Save the relevant info
2200                          * so we can convert this flow when that group
2201                          * materializes.
2202                          */
2203                         save_jump_flow(fm, flow, fet->group,
2204                                        fm_tcam_entry, fm_action);
2205                 } else if (fet && fet->ref == 1) {
2206                         /*
2207                          * A new table is created. Convert the saved flows
2208                          * that should jump to this group.
2209                          */
2210                         convert_jump_flows(fm, fet, error);
2211                 }
2212         }
2213
2214 error_with_scratch:
2215         enic_fm_close_scratch(fm);
2216         return flow;
2217 }
2218
2219 static int
2220 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2221                      __rte_unused struct rte_flow_error *error)
2222 {
2223         struct enic *enic = pmd_priv(dev);
2224
2225         ENICPMD_FUNC_TRACE();
2226         if (enic->fm == NULL)
2227                 return 0;
2228         LIST_REMOVE(flow, next);
2229         enic_fm_flow_free(enic->fm, flow);
2230         return 0;
2231 }
2232
2233 static int
2234 enic_fm_flow_flush(struct rte_eth_dev *dev,
2235                    __rte_unused struct rte_flow_error *error)
2236 {
2237         struct enic_fm_flow *fm_flow;
2238         struct enic_flowman *fm;
2239         struct rte_flow *flow;
2240         struct enic *enic = pmd_priv(dev);
2241
2242         ENICPMD_FUNC_TRACE();
2243         if (enic->fm == NULL)
2244                 return 0;
2245         fm = enic->fm;
2246         while (!LIST_EMPTY(&enic->flows)) {
2247                 flow = LIST_FIRST(&enic->flows);
2248                 fm_flow = flow->fm;
2249                 LIST_REMOVE(flow, next);
2250                 /*
2251                  * If tables are null, then vNIC is closing, and the firmware
2252                  * has already cleaned up flowman state. So do not try to free
2253                  * resources, as it only causes errors.
2254                  */
2255                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2256                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2257                         fm_flow->action_handle = FM_INVALID_HANDLE;
2258                         fm_flow->fet = NULL;
2259                 }
2260                 enic_fm_flow_free(fm, flow);
2261         }
2262         return 0;
2263 }
2264
2265 static int
2266 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2267 {
2268         uint64_t args[2];
2269         int rc;
2270
2271         args[0] = FM_MATCH_TABLE_FREE;
2272         args[1] = handle;
2273         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
2274         if (rc)
2275                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2276                             rc, handle);
2277         return rc;
2278 }
2279
2280 static int
2281 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2282                         uint32_t max_entries, uint64_t *handle)
2283 {
2284         struct fm_tcam_match_table *tcam_tbl;
2285         struct enic *enic;
2286         uint64_t args[2];
2287         int rc;
2288
2289         ENICPMD_FUNC_TRACE();
2290         enic = fm->enic;
2291         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2292         tcam_tbl->ftt_direction = direction;
2293         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2294         tcam_tbl->ftt_max_entries = max_entries;
2295         args[0] = FM_TCAM_TABLE_ALLOC;
2296         args[1] = fm->cmd.pa;
2297         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2298         if (rc) {
2299                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2300                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2301                 return rc;
2302         }
2303         *handle = args[0];
2304         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2305                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2306         return 0;
2307 }
2308
2309 static int
2310 enic_fm_init_counters(struct enic_flowman *fm)
2311 {
2312         ENICPMD_FUNC_TRACE();
2313         SLIST_INIT(&fm->counters);
2314         return enic_fm_more_counters(fm);
2315 }
2316
2317 static void
2318 enic_fm_free_all_counters(struct enic_flowman *fm)
2319 {
2320         struct enic *enic;
2321         uint64_t args[2];
2322         int rc;
2323
2324         enic = fm->enic;
2325         args[0] = FM_COUNTER_BRK;
2326         args[1] = 0;
2327         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2328         if (rc != 0)
2329                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2330         rte_free(fm->counter_stack);
2331 }
2332
2333 static int
2334 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2335 {
2336         int rc;
2337
2338         ENICPMD_FUNC_TRACE();
2339         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2340                                     &fm->ig_tcam_hndl);
2341         if (rc)
2342                 return rc;
2343         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2344                                     &fm->eg_tcam_hndl);
2345         return rc;
2346 }
2347
2348 static void
2349 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2350 {
2351         ENICPMD_FUNC_TRACE();
2352         if (fm->ig_tcam_hndl) {
2353                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2354                             fm->ig_tcam_hndl);
2355                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2356                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2357         }
2358         if (fm->eg_tcam_hndl) {
2359                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2360                             fm->eg_tcam_hndl);
2361                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2362                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2363         }
2364 }
2365
2366 int
2367 enic_fm_init(struct enic *enic)
2368 {
2369         struct enic_flowman *fm;
2370         uint8_t name[RTE_MEMZONE_NAMESIZE];
2371         int rc;
2372
2373         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2374                 return 0;
2375         ENICPMD_FUNC_TRACE();
2376         fm = calloc(1, sizeof(*fm));
2377         if (fm == NULL) {
2378                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2379                 return -ENOMEM;
2380         }
2381         fm->enic = enic;
2382         TAILQ_INIT(&fm->fet_list);
2383         TAILQ_INIT(&fm->jump_list);
2384         /* Allocate host memory for flowman commands */
2385         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2386         fm->cmd.va = enic_alloc_consistent(enic,
2387                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2388         if (!fm->cmd.va) {
2389                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2390                 rc = -ENOMEM;
2391                 goto error_fm;
2392         }
2393         /* Allocate TCAM tables upfront as they are the main tables */
2394         rc = enic_fm_alloc_tcam_tables(fm);
2395         if (rc) {
2396                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2397                 goto error_cmd;
2398         }
2399         /* Then a number of counters */
2400         rc = enic_fm_init_counters(fm);
2401         if (rc) {
2402                 ENICPMD_LOG(ERR, "cannot alloc counters");
2403                 goto error_tables;
2404         }
2405         /*
2406          * One default exact match table for each direction. We hold onto
2407          * it until close.
2408          */
2409         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2410         if (rc) {
2411                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2412                 goto error_counters;
2413         }
2414         fm->default_ig_fet->ref = 1;
2415         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2416         if (rc) {
2417                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2418                 goto error_ig_fet;
2419         }
2420         fm->default_eg_fet->ref = 1;
2421         enic->fm = fm;
2422         return 0;
2423
2424 error_ig_fet:
2425         enic_fet_free(fm, fm->default_ig_fet);
2426 error_counters:
2427         enic_fm_free_all_counters(fm);
2428 error_tables:
2429         enic_fm_free_tcam_tables(fm);
2430 error_cmd:
2431         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2432                 fm->cmd.va, fm->cmd.pa);
2433 error_fm:
2434         free(fm);
2435         return rc;
2436 }
2437
2438 void
2439 enic_fm_destroy(struct enic *enic)
2440 {
2441         struct enic_flowman *fm;
2442         struct enic_fm_fet *fet;
2443
2444         if (enic->fm == NULL)
2445                 return;
2446         ENICPMD_FUNC_TRACE();
2447         fm = enic->fm;
2448         enic_fet_free(fm, fm->default_eg_fet);
2449         enic_fet_free(fm, fm->default_ig_fet);
2450         /* Free all exact match tables still open */
2451         while (!TAILQ_EMPTY(&fm->fet_list)) {
2452                 fet = TAILQ_FIRST(&fm->fet_list);
2453                 enic_fet_free(fm, fet);
2454         }
2455         enic_fm_free_tcam_tables(fm);
2456         enic_fm_free_all_counters(fm);
2457         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2458                 fm->cmd.va, fm->cmd.pa);
2459         fm->cmd.va = NULL;
2460         free(fm);
2461         enic->fm = NULL;
2462 }
2463
2464 const struct rte_flow_ops enic_fm_flow_ops = {
2465         .validate = enic_fm_flow_validate,
2466         .create = enic_fm_flow_create,
2467         .destroy = enic_fm_flow_destroy,
2468         .flush = enic_fm_flow_flush,
2469         .query = enic_fm_flow_query,
2470 };