drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_ip.h>
  12 #include <rte_udp.h>
  13
  14 #include "enic_compat.h"
  15 #include "enic.h"
  16 #include "vnic_dev.h"
  17 #include "vnic_nic.h"
  18
  19 #define IP_DEFTTL  64   /* from RFC 1340. */
  20 #define IP_VERSION 0x40
  21 #define IP_HDRLEN  0x05 /* default IP header length == five 32-bits words. */
  22 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
  23 #define IP6_VTC_FLOW 0x60000000
  24
  25 /* Highest Item type supported by Flowman */
  26 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  27
  28 /* Up to 1024 TCAM entries */
  29 #define FM_MAX_TCAM_TABLE_SIZE 1024
  30
  31 /* Up to 4096 entries per exact match table */
  32 #define FM_MAX_EXACT_TABLE_SIZE 4096
  33
  34 /* Number of counters to increase on for each increment */
  35 #define FM_COUNTERS_EXPAND  100
  36
  37 #define FM_INVALID_HANDLE 0
  38
  39 /*
  40  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  41  * Use a simple scheme to map groups to tables.
  42  * Group 0 uses the single TCAM tables, one for each direction.
  43  * Group 1, 2, ... uses its own exact match table.
  44  *
  45  * The TCAM tables are allocated upfront during init.
  46  *
  47  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  48  *
  49  * 1. Add a flow that jumps from group 0 to group N.
  50  *
  51  * If N does not exist, we allocate an exact match table for it, using
  52  * a dummy key. A key is required for the table.
  53  *
  54  * 2. Add a flow that uses group N.
  55  *
  56  * If N does not exist, we allocate an exact match table for it, using
  57  * the flow's key. Subsequent flows to the same group all should have
  58  * the same key.
  59  *
  60  * Without a jump flow to N, N is not reachable in hardware. No packets
  61  * reach N and match.
  62  *
  63  * 3. Add a flow to an empty group N.
  64  *
  65  * N has been created via (1) and the dummy key. We free that table, allocate
  66  * a new table using the new flow's key. Also re-do the existing jump flow to
  67  * point to the new table.
  68  */
  69 #define FM_TCAM_RTE_GROUP 0
  70
  71 struct enic_fm_fet {
  72         TAILQ_ENTRY(enic_fm_fet) list;
  73         uint32_t group; /* rte_flow group ID */
  74         uint64_t handle; /* Exact match table handle from flowman */
  75         uint8_t ingress;
  76         uint8_t default_key;
  77         int ref; /* Reference count via get/put */
  78         struct fm_key_template key; /* Key associated with the table */
  79 };
  80
  81 struct enic_fm_counter {
  82         SLIST_ENTRY(enic_fm_counter) next;
  83         uint32_t handle;
  84 };
  85
  86 /* rte_flow.fm */
  87 struct enic_fm_flow {
  88         bool counter_valid;
  89         uint64_t entry_handle;
  90         uint64_t action_handle;
  91         struct enic_fm_counter *counter;
  92         struct enic_fm_fet *fet;
  93 };
  94
  95 struct enic_fm_jump_flow {
  96         TAILQ_ENTRY(enic_fm_jump_flow) list;
  97         struct rte_flow *flow;
  98         uint32_t group;
  99         struct fm_tcam_match_entry match;
 100         struct fm_action action;
 101 };
 102
 103 /*
 104  * Flowman uses host memory for commands. This structure is allocated
 105  * in DMA-able memory.
 106  */
 107 union enic_flowman_cmd_mem {
 108         struct fm_tcam_match_table fm_tcam_match_table;
 109         struct fm_exact_match_table fm_exact_match_table;
 110         struct fm_tcam_match_entry fm_tcam_match_entry;
 111         struct fm_exact_match_entry fm_exact_match_entry;
 112         struct fm_action fm_action;
 113 };
 114
 115 struct enic_flowman {
 116         struct enic *enic;
 117         /* Command buffer */
 118         struct {
 119                 union enic_flowman_cmd_mem *va;
 120                 dma_addr_t pa;
 121         } cmd;
 122         /* TCAM tables allocated upfront, used for group 0 */
 123         uint64_t ig_tcam_hndl;
 124         uint64_t eg_tcam_hndl;
 125         /* Counters */
 126         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 127         void *counter_stack;
 128         uint32_t counters_alloced;
 129         /* Exact match tables for groups != 0, dynamically allocated */
 130         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 131         /*
 132          * Default exact match tables used for jump actions to
 133          * non-existent groups.
 134          */
 135         struct enic_fm_fet *default_eg_fet;
 136         struct enic_fm_fet *default_ig_fet;
 137         /* Flows that jump to the default table above */
 138         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 139         /*
 140          * Scratch data used during each invocation of flow_create
 141          * and flow_validate.
 142          */
 143         struct enic_fm_fet *fet;
 144         struct fm_tcam_match_entry tcam_entry;
 145         struct fm_action action;
 146         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 147         int action_op_count;
 148 };
 149
 150 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 151
 152 /*
 153  * Common arguments passed to copy_item functions. Use this structure
 154  * so we can easily add new arguments.
 155  * item: Item specification.
 156  * fm_tcam_entry: Flowman TCAM match entry.
 157  * header_level: 0 for outer header, 1 for inner header.
 158  */
 159 struct copy_item_args {
 160         const struct rte_flow_item *item;
 161         struct fm_tcam_match_entry *fm_tcam_entry;
 162         uint8_t header_level;
 163 };
 164
 165 /* functions for copying items into flowman match */
 166 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 167
 168 /* Info about how to copy items into flowman match */
 169 struct enic_fm_items {
 170         /* Function for copying and validating an item. */
 171         enic_copy_item_fn * const copy_item;
 172         /* List of valid previous items. */
 173         const enum rte_flow_item_type * const prev_items;
 174         /*
 175          * True if it's OK for this item to be the first item. For some NIC
 176          * versions, it's invalid to start the stack above layer 3.
 177          */
 178         const uint8_t valid_start_item;
 179 };
 180
 181 static enic_copy_item_fn enic_fm_copy_item_eth;
 182 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 183 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 184 static enic_copy_item_fn enic_fm_copy_item_raw;
 185 static enic_copy_item_fn enic_fm_copy_item_sctp;
 186 static enic_copy_item_fn enic_fm_copy_item_tcp;
 187 static enic_copy_item_fn enic_fm_copy_item_udp;
 188 static enic_copy_item_fn enic_fm_copy_item_vlan;
 189 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 190
 191 /* Ingress actions */
 192 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 193         RTE_FLOW_ACTION_TYPE_COUNT,
 194         RTE_FLOW_ACTION_TYPE_DROP,
 195         RTE_FLOW_ACTION_TYPE_FLAG,
 196         RTE_FLOW_ACTION_TYPE_JUMP,
 197         RTE_FLOW_ACTION_TYPE_MARK,
 198         RTE_FLOW_ACTION_TYPE_PORT_ID,
 199         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 200         RTE_FLOW_ACTION_TYPE_QUEUE,
 201         RTE_FLOW_ACTION_TYPE_RSS,
 202         RTE_FLOW_ACTION_TYPE_VOID,
 203         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 204         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 205         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 206 };
 207
 208 /* Egress actions */
 209 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 210         RTE_FLOW_ACTION_TYPE_COUNT,
 211         RTE_FLOW_ACTION_TYPE_DROP,
 212         RTE_FLOW_ACTION_TYPE_JUMP,
 213         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 214         RTE_FLOW_ACTION_TYPE_VOID,
 215         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 216         RTE_FLOW_ACTION_TYPE_END,
 217 };
 218
 219 static const struct enic_fm_items enic_fm_items[] = {
 220         [RTE_FLOW_ITEM_TYPE_RAW] = {
 221                 .copy_item = enic_fm_copy_item_raw,
 222                 .valid_start_item = 0,
 223                 .prev_items = (const enum rte_flow_item_type[]) {
 224                                RTE_FLOW_ITEM_TYPE_UDP,
 225                                RTE_FLOW_ITEM_TYPE_END,
 226                 },
 227         },
 228         [RTE_FLOW_ITEM_TYPE_ETH] = {
 229                 .copy_item = enic_fm_copy_item_eth,
 230                 .valid_start_item = 1,
 231                 .prev_items = (const enum rte_flow_item_type[]) {
 232                                RTE_FLOW_ITEM_TYPE_END,
 233                 },
 234         },
 235         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 236                 .copy_item = enic_fm_copy_item_vlan,
 237                 .valid_start_item = 1,
 238                 .prev_items = (const enum rte_flow_item_type[]) {
 239                                RTE_FLOW_ITEM_TYPE_ETH,
 240                                RTE_FLOW_ITEM_TYPE_END,
 241                 },
 242         },
 243         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 244                 .copy_item = enic_fm_copy_item_ipv4,
 245                 .valid_start_item = 1,
 246                 .prev_items = (const enum rte_flow_item_type[]) {
 247                                RTE_FLOW_ITEM_TYPE_ETH,
 248                                RTE_FLOW_ITEM_TYPE_VLAN,
 249                                RTE_FLOW_ITEM_TYPE_END,
 250                 },
 251         },
 252         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 253                 .copy_item = enic_fm_copy_item_ipv6,
 254                 .valid_start_item = 1,
 255                 .prev_items = (const enum rte_flow_item_type[]) {
 256                                RTE_FLOW_ITEM_TYPE_ETH,
 257                                RTE_FLOW_ITEM_TYPE_VLAN,
 258                                RTE_FLOW_ITEM_TYPE_END,
 259                 },
 260         },
 261         [RTE_FLOW_ITEM_TYPE_UDP] = {
 262                 .copy_item = enic_fm_copy_item_udp,
 263                 .valid_start_item = 1,
 264                 .prev_items = (const enum rte_flow_item_type[]) {
 265                                RTE_FLOW_ITEM_TYPE_IPV4,
 266                                RTE_FLOW_ITEM_TYPE_IPV6,
 267                                RTE_FLOW_ITEM_TYPE_END,
 268                 },
 269         },
 270         [RTE_FLOW_ITEM_TYPE_TCP] = {
 271                 .copy_item = enic_fm_copy_item_tcp,
 272                 .valid_start_item = 1,
 273                 .prev_items = (const enum rte_flow_item_type[]) {
 274                                RTE_FLOW_ITEM_TYPE_IPV4,
 275                                RTE_FLOW_ITEM_TYPE_IPV6,
 276                                RTE_FLOW_ITEM_TYPE_END,
 277                 },
 278         },
 279         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 280                 .copy_item = enic_fm_copy_item_sctp,
 281                 .valid_start_item = 0,
 282                 .prev_items = (const enum rte_flow_item_type[]) {
 283                                RTE_FLOW_ITEM_TYPE_IPV4,
 284                                RTE_FLOW_ITEM_TYPE_IPV6,
 285                                RTE_FLOW_ITEM_TYPE_END,
 286                 },
 287         },
 288         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 289                 .copy_item = enic_fm_copy_item_vxlan,
 290                 .valid_start_item = 1,
 291                 .prev_items = (const enum rte_flow_item_type[]) {
 292                                RTE_FLOW_ITEM_TYPE_UDP,
 293                                RTE_FLOW_ITEM_TYPE_END,
 294                 },
 295         },
 296 };
 297
 298 static int
 299 enic_fm_copy_item_eth(struct copy_item_args *arg)
 300 {
 301         const struct rte_flow_item *item = arg->item;
 302         const struct rte_flow_item_eth *spec = item->spec;
 303         const struct rte_flow_item_eth *mask = item->mask;
 304         const uint8_t lvl = arg->header_level;
 305         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 306         struct fm_header_set *fm_data, *fm_mask;
 307
 308         ENICPMD_FUNC_TRACE();
 309         /* Match all if no spec */
 310         if (!spec)
 311                 return 0;
 312         if (!mask)
 313                 mask = &rte_flow_item_eth_mask;
 314         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 315         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 316         fm_data->fk_header_select |= FKH_ETHER;
 317         fm_mask->fk_header_select |= FKH_ETHER;
 318         memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
 319         memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
 320         return 0;
 321 }
 322
 323 static int
 324 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 325 {
 326         const struct rte_flow_item *item = arg->item;
 327         const struct rte_flow_item_vlan *spec = item->spec;
 328         const struct rte_flow_item_vlan *mask = item->mask;
 329         const uint8_t lvl = arg->header_level;
 330         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 331         struct fm_header_set *fm_data, *fm_mask;
 332         struct rte_ether_hdr *eth_mask;
 333         struct rte_ether_hdr *eth_val;
 334         uint32_t meta;
 335
 336         ENICPMD_FUNC_TRACE();
 337         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 338         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 339         /* Outer and inner packet vlans need different flags */
 340         meta = FKM_VLAN_PRES;
 341         if (lvl > 0)
 342                 meta = FKM_QTAG;
 343         fm_data->fk_metadata |= meta;
 344         fm_mask->fk_metadata |= meta;
 345
 346         /* Match all if no spec */
 347         if (!spec)
 348                 return 0;
 349         if (!mask)
 350                 mask = &rte_flow_item_vlan_mask;
 351
 352         eth_mask = (void *)&fm_mask->l2.eth;
 353         eth_val = (void *)&fm_data->l2.eth;
 354
 355         /* Outer TPID cannot be matched */
 356         if (eth_mask->ether_type)
 357                 return -ENOTSUP;
 358
 359         /*
 360          * When packet matching, the VIC always compares vlan-stripped
 361          * L2, regardless of vlan stripping settings. So, the inner type
 362          * from vlan becomes the ether type of the eth header.
 363          */
 364         eth_mask->ether_type = mask->inner_type;
 365         eth_val->ether_type = spec->inner_type;
 366         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 367         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 368         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 369         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 370         return 0;
 371 }
 372
 373 static int
 374 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 375 {
 376         const struct rte_flow_item *item = arg->item;
 377         const struct rte_flow_item_ipv4 *spec = item->spec;
 378         const struct rte_flow_item_ipv4 *mask = item->mask;
 379         const uint8_t lvl = arg->header_level;
 380         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 381         struct fm_header_set *fm_data, *fm_mask;
 382
 383         ENICPMD_FUNC_TRACE();
 384         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 385         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 386         fm_data->fk_metadata |= FKM_IPV4;
 387         fm_mask->fk_metadata |= FKM_IPV4;
 388
 389         if (!spec)
 390                 return 0;
 391         if (!mask)
 392                 mask = &rte_flow_item_ipv4_mask;
 393
 394         fm_data->fk_header_select |= FKH_IPV4;
 395         fm_mask->fk_header_select |= FKH_IPV4;
 396         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 397         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 398         return 0;
 399 }
 400
 401 static int
 402 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 403 {
 404         const struct rte_flow_item *item = arg->item;
 405         const struct rte_flow_item_ipv6 *spec = item->spec;
 406         const struct rte_flow_item_ipv6 *mask = item->mask;
 407         const uint8_t lvl = arg->header_level;
 408         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 409         struct fm_header_set *fm_data, *fm_mask;
 410
 411         ENICPMD_FUNC_TRACE();
 412         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 413         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 414         fm_data->fk_metadata |= FKM_IPV6;
 415         fm_mask->fk_metadata |= FKM_IPV6;
 416
 417         if (!spec)
 418                 return 0;
 419         if (!mask)
 420                 mask = &rte_flow_item_ipv6_mask;
 421
 422         fm_data->fk_header_select |= FKH_IPV6;
 423         fm_mask->fk_header_select |= FKH_IPV6;
 424         memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
 425         memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
 426         return 0;
 427 }
 428
 429 static int
 430 enic_fm_copy_item_udp(struct copy_item_args *arg)
 431 {
 432         const struct rte_flow_item *item = arg->item;
 433         const struct rte_flow_item_udp *spec = item->spec;
 434         const struct rte_flow_item_udp *mask = item->mask;
 435         const uint8_t lvl = arg->header_level;
 436         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 437         struct fm_header_set *fm_data, *fm_mask;
 438
 439         ENICPMD_FUNC_TRACE();
 440         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 441         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 442         fm_data->fk_metadata |= FKM_UDP;
 443         fm_mask->fk_metadata |= FKM_UDP;
 444
 445         if (!spec)
 446                 return 0;
 447         if (!mask)
 448                 mask = &rte_flow_item_udp_mask;
 449
 450         fm_data->fk_header_select |= FKH_UDP;
 451         fm_mask->fk_header_select |= FKH_UDP;
 452         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 453         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 454         return 0;
 455 }
 456
 457 static int
 458 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 459 {
 460         const struct rte_flow_item *item = arg->item;
 461         const struct rte_flow_item_tcp *spec = item->spec;
 462         const struct rte_flow_item_tcp *mask = item->mask;
 463         const uint8_t lvl = arg->header_level;
 464         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 465         struct fm_header_set *fm_data, *fm_mask;
 466
 467         ENICPMD_FUNC_TRACE();
 468         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 469         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 470         fm_data->fk_metadata |= FKM_TCP;
 471         fm_mask->fk_metadata |= FKM_TCP;
 472
 473         if (!spec)
 474                 return 0;
 475         if (!mask)
 476                 mask = &rte_flow_item_tcp_mask;
 477
 478         fm_data->fk_header_select |= FKH_TCP;
 479         fm_mask->fk_header_select |= FKH_TCP;
 480         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 481         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 482         return 0;
 483 }
 484
 485 static int
 486 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 487 {
 488         const struct rte_flow_item *item = arg->item;
 489         const struct rte_flow_item_sctp *spec = item->spec;
 490         const struct rte_flow_item_sctp *mask = item->mask;
 491         const uint8_t lvl = arg->header_level;
 492         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 493         struct fm_header_set *fm_data, *fm_mask;
 494         uint8_t *ip_proto_mask = NULL;
 495         uint8_t *ip_proto = NULL;
 496         uint32_t l3_fkh;
 497
 498         ENICPMD_FUNC_TRACE();
 499         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 500         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 501         /*
 502          * The NIC filter API has no flags for "match sctp", so explicitly
 503          * set the protocol number in the IP pattern.
 504          */
 505         if (fm_data->fk_metadata & FKM_IPV4) {
 506                 struct rte_ipv4_hdr *ip;
 507                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 508                 ip_proto_mask = &ip->next_proto_id;
 509                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 510                 ip_proto = &ip->next_proto_id;
 511                 l3_fkh = FKH_IPV4;
 512         } else if (fm_data->fk_metadata & FKM_IPV6) {
 513                 struct rte_ipv6_hdr *ip;
 514                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 515                 ip_proto_mask = &ip->proto;
 516                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 517                 ip_proto = &ip->proto;
 518                 l3_fkh = FKH_IPV6;
 519         } else {
 520                 /* Need IPv4/IPv6 pattern first */
 521                 return -EINVAL;
 522         }
 523         *ip_proto = IPPROTO_SCTP;
 524         *ip_proto_mask = 0xff;
 525         fm_data->fk_header_select |= l3_fkh;
 526         fm_mask->fk_header_select |= l3_fkh;
 527
 528         if (!spec)
 529                 return 0;
 530         if (!mask)
 531                 mask = &rte_flow_item_sctp_mask;
 532
 533         fm_data->fk_header_select |= FKH_L4RAW;
 534         fm_mask->fk_header_select |= FKH_L4RAW;
 535         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 536         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 537         return 0;
 538 }
 539
 540 static int
 541 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 542 {
 543         const struct rte_flow_item *item = arg->item;
 544         const struct rte_flow_item_vxlan *spec = item->spec;
 545         const struct rte_flow_item_vxlan *mask = item->mask;
 546         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 547         struct fm_header_set *fm_data, *fm_mask;
 548
 549         ENICPMD_FUNC_TRACE();
 550         /* Only 2 header levels (outer and inner) allowed */
 551         if (arg->header_level > 0)
 552                 return -EINVAL;
 553
 554         fm_data = &entry->ftm_data.fk_hdrset[0];
 555         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 556         fm_data->fk_metadata |= FKM_VXLAN;
 557         fm_mask->fk_metadata |= FKM_VXLAN;
 558         /* items from here on out are inner header items */
 559         arg->header_level = 1;
 560
 561         /* Match all if no spec */
 562         if (!spec)
 563                 return 0;
 564         if (!mask)
 565                 mask = &rte_flow_item_vxlan_mask;
 566
 567         fm_data->fk_header_select |= FKH_VXLAN;
 568         fm_mask->fk_header_select |= FKH_VXLAN;
 569         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 570         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 571         return 0;
 572 }
 573
 574 /*
 575  * Currently, raw pattern match is very limited. It is intended for matching
 576  * UDP tunnel header (e.g. vxlan or geneve).
 577  */
 578 static int
 579 enic_fm_copy_item_raw(struct copy_item_args *arg)
 580 {
 581         const struct rte_flow_item *item = arg->item;
 582         const struct rte_flow_item_raw *spec = item->spec;
 583         const struct rte_flow_item_raw *mask = item->mask;
 584         const uint8_t lvl = arg->header_level;
 585         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 586         struct fm_header_set *fm_data, *fm_mask;
 587
 588         ENICPMD_FUNC_TRACE();
 589         /* Cannot be used for inner packet */
 590         if (lvl > 0)
 591                 return -EINVAL;
 592         /* Need both spec and mask */
 593         if (!spec || !mask)
 594                 return -EINVAL;
 595         /* Only supports relative with offset 0 */
 596         if (!spec->relative || spec->offset != 0 || spec->search ||
 597             spec->limit)
 598                 return -EINVAL;
 599         /* Need non-null pattern that fits within the NIC's filter pattern */
 600         if (spec->length == 0 ||
 601             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 602             !spec->pattern || !mask->pattern)
 603                 return -EINVAL;
 604         /*
 605          * Mask fields, including length, are often set to zero. Assume that
 606          * means "same as spec" to avoid breaking existing apps. If length
 607          * is not zero, then it should be >= spec length.
 608          *
 609          * No more pattern follows this, so append to the L4 layer instead of
 610          * L5 to work with both recent and older VICs.
 611          */
 612         if (mask->length != 0 && mask->length < spec->length)
 613                 return -EINVAL;
 614
 615         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 616         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 617         fm_data->fk_header_select |= FKH_L4RAW;
 618         fm_mask->fk_header_select |= FKH_L4RAW;
 619         fm_data->fk_header_select &= ~FKH_UDP;
 620         fm_mask->fk_header_select &= ~FKH_UDP;
 621         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 622                spec->pattern, spec->length);
 623         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 624                mask->pattern, spec->length);
 625         return 0;
 626 }
 627
 628 static int
 629 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 630                struct fm_key_template *key, int entries,
 631                struct enic_fm_fet **fet_out)
 632 {
 633         struct fm_exact_match_table *cmd;
 634         struct fm_header_set *hdr;
 635         struct enic_fm_fet *fet;
 636         u64 args[3];
 637         int ret;
 638
 639         ENICPMD_FUNC_TRACE();
 640         fet = calloc(1, sizeof(struct enic_fm_fet));
 641         if (fet == NULL)
 642                 return -ENOMEM;
 643         cmd = &fm->cmd.va->fm_exact_match_table;
 644         memset(cmd, 0, sizeof(*cmd));
 645         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 646         cmd->fet_stage = FM_STAGE_LAST;
 647         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 648         if (key == NULL) {
 649                 hdr = &cmd->fet_key.fk_hdrset[0];
 650                 memset(hdr, 0, sizeof(*hdr));
 651                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 652                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 653                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 654                 hdr->l4.udp.fk_source = 0xFFFF;
 655                 hdr->l4.udp.fk_dest = 0xFFFF;
 656                 fet->default_key = 1;
 657         } else {
 658                 memcpy(&cmd->fet_key, key, sizeof(*key));
 659                 memcpy(&fet->key, key, sizeof(*key));
 660                 fet->default_key = 0;
 661         }
 662         cmd->fet_key.fk_packet_tag = 1;
 663
 664         args[0] = FM_EXACT_TABLE_ALLOC;
 665         args[1] = fm->cmd.pa;
 666         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
 667         if (ret) {
 668                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 669                 free(fet);
 670                 return ret;
 671         }
 672         fet->handle = args[0];
 673         fet->ingress = ingress;
 674         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 675                     fet->handle);
 676         *fet_out = fet;
 677         return 0;
 678 }
 679
 680 static void
 681 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 682 {
 683         ENICPMD_FUNC_TRACE();
 684         enic_fm_tbl_free(fm, fet->handle);
 685         if (!fet->default_key)
 686                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 687         free(fet);
 688 }
 689
 690 /*
 691  * Get the exact match table for the given combination of
 692  * <group, ingress, key>. Allocate one on the fly as necessary.
 693  */
 694 static int
 695 enic_fet_get(struct enic_flowman *fm,
 696              uint32_t group,
 697              uint8_t ingress,
 698              struct fm_key_template *key,
 699              struct enic_fm_fet **fet_out,
 700              struct rte_flow_error *error)
 701 {
 702         struct enic_fm_fet *fet;
 703
 704         ENICPMD_FUNC_TRACE();
 705         /* See if we already have this table open */
 706         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 707                 if (fet->group == group && fet->ingress == ingress)
 708                         break;
 709         }
 710         if (fet == NULL) {
 711                 /* Jumping to a non-existing group? Use the default table */
 712                 if (key == NULL) {
 713                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 714                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 715                         return rte_flow_error_set(error, EINVAL,
 716                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 717                                 NULL, "enic: cannot get exact match table");
 718                 }
 719                 fet->group = group;
 720                 /* Default table is never on the open table list */
 721                 if (!fet->default_key)
 722                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 723         }
 724         fet->ref++;
 725         *fet_out = fet;
 726         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 727                     fet->default_key ? "default" : "",
 728                     fet->ingress ? "ingress" : "egress",
 729                     fet->group, fet->ref);
 730         return 0;
 731 }
 732
 733 static void
 734 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 735 {
 736         ENICPMD_FUNC_TRACE();
 737         RTE_ASSERT(fet->ref > 0);
 738         fet->ref--;
 739         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 740                     fet->default_key ? "default" : "",
 741                     fet->ingress ? "ingress" : "egress",
 742                     fet->group, fet->ref);
 743         if (fet->ref == 0)
 744                 enic_fet_free(fm, fet);
 745 }
 746
 747 /* Return 1 if current item is valid on top of the previous one. */
 748 static int
 749 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 750                        const struct enic_fm_items *item_info,
 751                        uint8_t is_first_item)
 752 {
 753         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 754
 755         ENICPMD_FUNC_TRACE();
 756         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 757                 if (prev_item == *allowed_items)
 758                         return 1;
 759         }
 760
 761         /* This is the first item in the stack. Check if that's cool */
 762         if (is_first_item && item_info->valid_start_item)
 763                 return 1;
 764         return 0;
 765 }
 766
 767 /*
 768  * Build the flow manager match entry structure from the provided pattern.
 769  * The pattern is validated as the items are copied.
 770  */
 771 static int
 772 enic_fm_copy_entry(struct enic_flowman *fm,
 773                    const struct rte_flow_item pattern[],
 774                    struct rte_flow_error *error)
 775 {
 776         const struct enic_fm_items *item_info;
 777         enum rte_flow_item_type prev_item;
 778         const struct rte_flow_item *item;
 779         struct copy_item_args args;
 780         uint8_t prev_header_level;
 781         uint8_t is_first_item;
 782         int ret;
 783
 784         ENICPMD_FUNC_TRACE();
 785         item = pattern;
 786         is_first_item = 1;
 787         prev_item = RTE_FLOW_ITEM_TYPE_END;
 788
 789         args.fm_tcam_entry = &fm->tcam_entry;
 790         args.header_level = 0;
 791         prev_header_level = 0;
 792         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 793                 /*
 794                  * Get info about how to validate and copy the item. If NULL
 795                  * is returned the nic does not support the item.
 796                  */
 797                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 798                         continue;
 799
 800                 item_info = &enic_fm_items[item->type];
 801
 802                 if (item->type > FM_MAX_ITEM_TYPE ||
 803                     item_info->copy_item == NULL) {
 804                         return rte_flow_error_set(error, ENOTSUP,
 805                                 RTE_FLOW_ERROR_TYPE_ITEM,
 806                                 NULL, "enic: unsupported item");
 807                 }
 808
 809                 /* check to see if item stacking is valid */
 810                 if (!fm_item_stacking_valid(prev_item, item_info,
 811                                             is_first_item))
 812                         goto stacking_error;
 813
 814                 args.item = item;
 815                 ret = item_info->copy_item(&args);
 816                 if (ret)
 817                         goto item_not_supported;
 818                 /* Going from outer to inner? Treat it as a new packet start */
 819                 if (prev_header_level != args.header_level) {
 820                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 821                         is_first_item = 1;
 822                 } else {
 823                         prev_item = item->type;
 824                         is_first_item = 0;
 825                 }
 826                 prev_header_level = args.header_level;
 827         }
 828         return 0;
 829
 830 item_not_supported:
 831         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 832                                   NULL, "enic: unsupported item type");
 833
 834 stacking_error:
 835         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 836                                   item, "enic: unsupported item stack");
 837 }
 838
 839 static void
 840 flow_item_skip_void(const struct rte_flow_item **item)
 841 {
 842         for ( ; ; (*item)++)
 843                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 844                         return;
 845 }
 846
 847 static void
 848 append_template(void **template, uint8_t *off, const void *data, int len)
 849 {
 850         memcpy(*template, data, len);
 851         *template = (char *)*template + len;
 852         *off = *off + len;
 853 }
 854
 855 static int
 856 enic_fm_append_action_op(struct enic_flowman *fm,
 857                          struct fm_action_op *fm_op,
 858                          struct rte_flow_error *error)
 859 {
 860         int count;
 861
 862         count = fm->action_op_count;
 863         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 864                     count, fm_op->fa_op);
 865         if (count == FM_ACTION_OP_MAX) {
 866                 return rte_flow_error_set(error, EINVAL,
 867                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 868                         "too many action operations");
 869         }
 870         fm->action.fma_action_ops[count] = *fm_op;
 871         fm->action_op_count = count + 1;
 872         return 0;
 873 }
 874
 875 /* Steer operations need to appear before other ops */
 876 static void
 877 enic_fm_reorder_action_op(struct enic_flowman *fm)
 878 {
 879         struct fm_action_op *dst, *dst_head, *src, *src_head;
 880
 881         ENICPMD_FUNC_TRACE();
 882         /* Move steer ops to the front. */
 883         src = fm->action.fma_action_ops;
 884         src_head = src;
 885         dst = fm->action_tmp.fma_action_ops;
 886         dst_head = dst;
 887         /* Copy steer ops to tmp */
 888         while (src->fa_op != FMOP_END) {
 889                 if (src->fa_op == FMOP_RQ_STEER) {
 890                         ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
 891                                     (long)(src - src_head),
 892                                     (long)(dst - dst_head));
 893                         *dst = *src;
 894                         dst++;
 895                 }
 896                 src++;
 897         }
 898         /* Then append non-steer ops */
 899         src = src_head;
 900         while (src->fa_op != FMOP_END) {
 901                 if (src->fa_op != FMOP_RQ_STEER) {
 902                         ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
 903                                     (long)(src - src_head),
 904                                     (long)(dst - dst_head));
 905                         *dst = *src;
 906                         dst++;
 907                 }
 908                 src++;
 909         }
 910         /* Copy END */
 911         *dst = *src;
 912         /* Finally replace the original action with the reordered one */
 913         memcpy(fm->action.fma_action_ops, fm->action_tmp.fma_action_ops,
 914                sizeof(fm->action.fma_action_ops));
 915 }
 916
 917 /* VXLAN decap is done via flowman compound action */
 918 static int
 919 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 920                          struct fm_tcam_match_entry *fmt,
 921                          const struct rte_flow_action *action,
 922                          struct rte_flow_error *error)
 923 {
 924         struct fm_header_set *fm_data;
 925         struct fm_action_op fm_op;
 926
 927         ENICPMD_FUNC_TRACE();
 928         fm_data = &fmt->ftm_data.fk_hdrset[0];
 929         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 930                 return rte_flow_error_set(error, EINVAL,
 931                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 932                         "vxlan-decap: vxlan must be in pattern");
 933         }
 934
 935         memset(&fm_op, 0, sizeof(fm_op));
 936         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
 937         return enic_fm_append_action_op(fm, &fm_op, error);
 938 }
 939
 940 /* VXLAN encap is done via flowman compound action */
 941 static int
 942 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
 943                          const struct rte_flow_item *item,
 944                          struct rte_flow_error *error)
 945 {
 946         struct fm_action_op fm_op;
 947         struct rte_ether_hdr *eth;
 948         uint16_t *ethertype;
 949         void *template;
 950         uint8_t off;
 951
 952         ENICPMD_FUNC_TRACE();
 953         memset(&fm_op, 0, sizeof(fm_op));
 954         fm_op.fa_op = FMOP_ENCAP;
 955         template = fm->action.fma_data;
 956         off = 0;
 957         /*
 958          * Copy flow items to the flowman template starting L2.
 959          * L2 must be ethernet.
 960          */
 961         flow_item_skip_void(&item);
 962         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
 963                 return rte_flow_error_set(error, EINVAL,
 964                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 965                         "vxlan-encap: first item should be ethernet");
 966         eth = (struct rte_ether_hdr *)template;
 967         ethertype = &eth->ether_type;
 968         append_template(&template, &off, item->spec,
 969                         sizeof(struct rte_flow_item_eth));
 970         item++;
 971         flow_item_skip_void(&item);
 972         /* Optional VLAN */
 973         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
 974                 const struct rte_flow_item_vlan *spec;
 975
 976                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
 977                 spec = item->spec;
 978                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
 979                 item++;
 980                 flow_item_skip_void(&item);
 981         }
 982         /* L3 must be IPv4, IPv6 */
 983         switch (item->type) {
 984         case RTE_FLOW_ITEM_TYPE_IPV4:
 985         {
 986                 struct rte_ipv4_hdr *ip4;
 987
 988                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
 989                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 990                 ip4 = (struct rte_ipv4_hdr *)template;
 991                 /*
 992                  * Offset of IPv4 length field and its initial value
 993                  * (IP + UDP + VXLAN) are specified in the action. The NIC
 994                  * will add inner packet length.
 995                  */
 996                 fm_op.encap.len1_offset = off +
 997                         offsetof(struct rte_ipv4_hdr, total_length);
 998                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
 999                         sizeof(struct rte_udp_hdr) +
1000                         sizeof(struct rte_vxlan_hdr);
1001                 append_template(&template, &off, item->spec,
1002                                 sizeof(struct rte_ipv4_hdr));
1003                 ip4->version_ihl = IP_VHL_DEF;
1004                 if (ip4->time_to_live == 0)
1005                         ip4->time_to_live = IP_DEFTTL;
1006                 ip4->next_proto_id = IPPROTO_UDP;
1007                 break;
1008         }
1009         case RTE_FLOW_ITEM_TYPE_IPV6:
1010         {
1011                 struct rte_ipv6_hdr *ip6;
1012
1013                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1014                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1015                 ip6 = (struct rte_ipv6_hdr *)template;
1016                 fm_op.encap.len1_offset = off +
1017                         offsetof(struct rte_ipv6_hdr, payload_len);
1018                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1019                         sizeof(struct rte_vxlan_hdr);
1020                 append_template(&template, &off, item->spec,
1021                                 sizeof(struct rte_ipv6_hdr));
1022                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1023                 if (ip6->hop_limits == 0)
1024                         ip6->hop_limits = IP_DEFTTL;
1025                 ip6->proto = IPPROTO_UDP;
1026                 break;
1027         }
1028         default:
1029                 return rte_flow_error_set(error,
1030                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1031                         "vxlan-encap: L3 must be IPv4/IPv6");
1032         }
1033         item++;
1034         flow_item_skip_void(&item);
1035
1036         /* L4 is UDP */
1037         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1038                 return rte_flow_error_set(error, EINVAL,
1039                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1040                         "vxlan-encap: UDP must follow IPv4/IPv6");
1041         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1042         fm_op.encap.len2_offset =
1043                 off + offsetof(struct rte_udp_hdr, dgram_len);
1044         fm_op.encap.len2_delta =
1045                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1046         append_template(&template, &off, item->spec,
1047                         sizeof(struct rte_udp_hdr));
1048         item++;
1049         flow_item_skip_void(&item);
1050
1051         /* Finally VXLAN */
1052         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1053                 return rte_flow_error_set(error,
1054                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1055                         "vxlan-encap: VXLAN must follow UDP");
1056         append_template(&template, &off, item->spec,
1057                         sizeof(struct rte_flow_item_vxlan));
1058
1059         /*
1060          * Fill in the rest of the action structure.
1061          * Indicate that we want to encap with vxlan at packet start.
1062          */
1063         fm_op.encap.template_offset = 0;
1064         fm_op.encap.template_len = off;
1065         return enic_fm_append_action_op(fm, &fm_op, error);
1066 }
1067
1068 static int
1069 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1070                   uint64_t *handle)
1071 {
1072         uint32_t bdf;
1073         u64 args[2];
1074         int rc;
1075
1076         ENICPMD_FUNC_TRACE();
1077         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1078                     addr->function);
1079         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1080         args[0] = FM_VNIC_FIND;
1081         args[1] = bdf;
1082         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1083         if (rc != 0) {
1084                 ENICPMD_LOG(ERR, "allocating counters rc=%d", rc);
1085                 return rc;
1086         }
1087         *handle = args[0];
1088         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1089         return 0;
1090 }
1091
1092 /* Translate flow actions to flowman TCAM entry actions */
1093 static int
1094 enic_fm_copy_action(struct enic_flowman *fm,
1095                     const struct rte_flow_action actions[],
1096                     uint8_t ingress,
1097                     struct rte_flow_error *error)
1098 {
1099         enum {
1100                 FATE = 1 << 0,
1101                 MARK = 1 << 1,
1102                 PASSTHRU = 1 << 2,
1103                 COUNT = 1 << 3,
1104                 ENCAP = 1 << 4,
1105         };
1106         struct fm_tcam_match_entry *fmt;
1107         struct fm_action_op fm_op;
1108         struct enic *enic;
1109         uint32_t overlap;
1110         uint64_t vnic_h;
1111         bool first_rq;
1112         int ret;
1113
1114         ENICPMD_FUNC_TRACE();
1115         fmt = &fm->tcam_entry;
1116         first_rq = true;
1117         enic = fm->enic;
1118         overlap = 0;
1119         vnic_h = 0; /* 0 = current vNIC */
1120         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1121                 switch (actions->type) {
1122                 case RTE_FLOW_ACTION_TYPE_VOID:
1123                         continue;
1124                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1125                         if (overlap & PASSTHRU)
1126                                 goto unsupported;
1127                         overlap |= PASSTHRU;
1128                         break;
1129                 }
1130                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1131                         const struct rte_flow_action_jump *jump =
1132                                 actions->conf;
1133                         struct enic_fm_fet *fet;
1134
1135                         if (overlap & FATE)
1136                                 goto unsupported;
1137                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1138                                            &fet, error);
1139                         if (ret)
1140                                 return ret;
1141                         overlap |= FATE;
1142                         memset(&fm_op, 0, sizeof(fm_op));
1143                         fm_op.fa_op = FMOP_EXACT_MATCH;
1144                         fm_op.exact.handle = fet->handle;
1145                         fm->fet = fet;
1146                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1147                         if (ret)
1148                                 return ret;
1149                         break;
1150                 }
1151                 case RTE_FLOW_ACTION_TYPE_MARK: {
1152                         const struct rte_flow_action_mark *mark =
1153                                 actions->conf;
1154
1155                         if (overlap & MARK)
1156                                 goto unsupported;
1157                         overlap |= MARK;
1158                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1159                                 return rte_flow_error_set(error, EINVAL,
1160                                         RTE_FLOW_ERROR_TYPE_ACTION,
1161                                         NULL, "invalid mark id");
1162                         memset(&fm_op, 0, sizeof(fm_op));
1163                         fm_op.fa_op = FMOP_MARK;
1164                         fm_op.mark.mark = mark->id + 1;
1165                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1166                         if (ret)
1167                                 return ret;
1168                         break;
1169                 }
1170                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1171                         if (overlap & MARK)
1172                                 goto unsupported;
1173                         overlap |= MARK;
1174                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1175                         memset(&fm_op, 0, sizeof(fm_op));
1176                         fm_op.fa_op = FMOP_MARK;
1177                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1178                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1179                         if (ret)
1180                                 return ret;
1181                         break;
1182                 }
1183                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1184                         const struct rte_flow_action_queue *queue =
1185                                 actions->conf;
1186
1187                         /*
1188                          * If other fate kind is set, fail.  Multiple
1189                          * queue actions are ok.
1190                          */
1191                         if ((overlap & FATE) && first_rq)
1192                                 goto unsupported;
1193                         first_rq = false;
1194                         overlap |= FATE;
1195                         memset(&fm_op, 0, sizeof(fm_op));
1196                         fm_op.fa_op = FMOP_RQ_STEER;
1197                         fm_op.rq_steer.rq_index =
1198                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1199                         fm_op.rq_steer.vnic_handle = vnic_h;
1200                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1201                         if (ret)
1202                                 return ret;
1203                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1204                                     fm_op.rq_steer.rq_index);
1205                         break;
1206                 }
1207                 case RTE_FLOW_ACTION_TYPE_DROP: {
1208                         if (overlap & FATE)
1209                                 goto unsupported;
1210                         overlap |= FATE;
1211                         memset(&fm_op, 0, sizeof(fm_op));
1212                         fm_op.fa_op = FMOP_DROP;
1213                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1214                         if (ret)
1215                                 return ret;
1216                         ENICPMD_LOG(DEBUG, "create DROP action");
1217                         break;
1218                 }
1219                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1220                         if (overlap & COUNT)
1221                                 goto unsupported;
1222                         overlap |= COUNT;
1223                         /* Count is associated with entry not action on VIC. */
1224                         fmt->ftm_flags |= FMEF_COUNTER;
1225                         break;
1226                 }
1227                 case RTE_FLOW_ACTION_TYPE_RSS: {
1228                         const struct rte_flow_action_rss *rss = actions->conf;
1229                         bool allow;
1230                         uint16_t i;
1231
1232                         /*
1233                          * Hardware does not support general RSS actions, but
1234                          * we can still support the dummy one that is used to
1235                          * "receive normally".
1236                          */
1237                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1238                                 rss->level == 0 &&
1239                                 (rss->types == 0 ||
1240                                  rss->types == enic->rss_hf) &&
1241                                 rss->queue_num == enic->rq_count &&
1242                                 rss->key_len == 0;
1243                         /* Identity queue map is ok */
1244                         for (i = 0; i < rss->queue_num; i++)
1245                                 allow = allow && (i == rss->queue[i]);
1246                         if (!allow)
1247                                 goto unsupported;
1248                         if (overlap & FATE)
1249                                 goto unsupported;
1250                         /* Need MARK or FLAG */
1251                         if (!(overlap & MARK))
1252                                 goto unsupported;
1253                         overlap |= FATE;
1254                         break;
1255                 }
1256                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1257                         const struct rte_flow_action_port_id *port;
1258                         struct rte_pci_device *pdev;
1259                         struct rte_eth_dev *dev;
1260
1261                         port = actions->conf;
1262                         if (port->original) {
1263                                 vnic_h = 0; /* This port */
1264                                 break;
1265                         }
1266                         ENICPMD_LOG(DEBUG, "port id %u", port->id);
1267                         if (!rte_eth_dev_is_valid_port(port->id)) {
1268                                 return rte_flow_error_set(error, EINVAL,
1269                                         RTE_FLOW_ERROR_TYPE_ACTION,
1270                                         NULL, "invalid port_id");
1271                         }
1272                         dev = &rte_eth_devices[port->id];
1273                         if (!dev_is_enic(dev)) {
1274                                 return rte_flow_error_set(error, EINVAL,
1275                                         RTE_FLOW_ERROR_TYPE_ACTION,
1276                                         NULL, "port_id is not enic");
1277                         }
1278                         pdev = RTE_ETH_DEV_TO_PCI(dev);
1279                         if (enic_fm_find_vnic(enic, &pdev->addr, &vnic_h)) {
1280                                 return rte_flow_error_set(error, EINVAL,
1281                                         RTE_FLOW_ERROR_TYPE_ACTION,
1282                                         NULL, "port_id is not vnic");
1283                         }
1284                         break;
1285                 }
1286                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1287                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1288                                 error);
1289                         if (ret != 0)
1290                                 return ret;
1291                         break;
1292                 }
1293                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1294                         const struct rte_flow_action_vxlan_encap *encap;
1295
1296                         encap = actions->conf;
1297                         if (overlap & ENCAP)
1298                                 goto unsupported;
1299                         overlap |= ENCAP;
1300                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1301                                 error);
1302                         if (ret != 0)
1303                                 return ret;
1304                         break;
1305                 }
1306                 default:
1307                         goto unsupported;
1308                 }
1309         }
1310
1311         if (!(overlap & (FATE | PASSTHRU | COUNT)))
1312                 goto unsupported;
1313         memset(&fm_op, 0, sizeof(fm_op));
1314         fm_op.fa_op = FMOP_END;
1315         ret = enic_fm_append_action_op(fm, &fm_op, error);
1316         if (ret)
1317                 return ret;
1318         enic_fm_reorder_action_op(fm);
1319         return 0;
1320
1321 unsupported:
1322         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1323                                   NULL, "enic: unsupported action");
1324 }
1325
1326 /** Check if the action is supported */
1327 static int
1328 enic_fm_match_action(const struct rte_flow_action *action,
1329                      const enum rte_flow_action_type *supported_actions)
1330 {
1331         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1332              supported_actions++) {
1333                 if (action->type == *supported_actions)
1334                         return 1;
1335         }
1336         return 0;
1337 }
1338
1339 /* Debug function to dump internal NIC action structure. */
1340 static void
1341 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1342 {
1343         /* Manually keep in sync with FMOP commands */
1344         const char *fmop_str[FMOP_OP_MAX] = {
1345                 [FMOP_END] = "end",
1346                 [FMOP_DROP] = "drop",
1347                 [FMOP_RQ_STEER] = "steer",
1348                 [FMOP_EXACT_MATCH] = "exmatch",
1349                 [FMOP_MARK] = "mark",
1350                 [FMOP_EXT_MARK] = "ext_mark",
1351                 [FMOP_TAG] = "tag",
1352                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1353                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1354                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1355                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1356                 [FMOP_ENCAP] = "encap",
1357                 [FMOP_SET_OVLAN] = "set_ovlan",
1358                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1359         };
1360         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1361         char buf[128], *bp = buf;
1362         const char *op_str;
1363         int i, n, buf_len;
1364
1365         buf[0] = '\0';
1366         buf_len = sizeof(buf);
1367         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1368                 if (op->fa_op == FMOP_END)
1369                         break;
1370                 if (op->fa_op >= FMOP_OP_MAX)
1371                         op_str = "unknown";
1372                 else
1373                         op_str = fmop_str[op->fa_op];
1374                 n = snprintf(bp, buf_len, "%s,", op_str);
1375                 if (n > 0 && n < buf_len) {
1376                         bp += n;
1377                         buf_len -= n;
1378                 }
1379                 op++;
1380         }
1381         /* Remove trailing comma */
1382         if (buf[0])
1383                 *(bp - 1) = '\0';
1384         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1385 }
1386
1387 static int
1388 bits_to_str(uint32_t bits, const char *strings[], int max,
1389             char *buf, int buf_len)
1390 {
1391         int i, n = 0, len = 0;
1392
1393         for (i = 0; i < max; i++) {
1394                 if (bits & (1 << i)) {
1395                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1396                         if (n > 0 && n < buf_len) {
1397                                 buf += n;
1398                                 buf_len -= n;
1399                                 len += n;
1400                         }
1401                 }
1402         }
1403         /* Remove trailing comma */
1404         if (len) {
1405                 *(buf - 1) = '\0';
1406                 len--;
1407         }
1408         return len;
1409 }
1410
1411 /* Debug function to dump internal NIC filter structure. */
1412 static void
1413 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1414                           int buf_len)
1415 {
1416         /* Manually keep in sync with FKM_BITS */
1417         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1418                 [FKM_QTAG_BIT] = "qtag",
1419                 [FKM_CMD_BIT] = "cmd",
1420                 [FKM_IPV4_BIT] = "ip4",
1421                 [FKM_IPV6_BIT] = "ip6",
1422                 [FKM_ROCE_BIT] = "roce",
1423                 [FKM_UDP_BIT] = "udp",
1424                 [FKM_TCP_BIT] = "tcp",
1425                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1426                 [FKM_IPFRAG_BIT] = "ipfrag",
1427                 [FKM_NVGRE_BIT] = "nvgre",
1428                 [FKM_VXLAN_BIT] = "vxlan",
1429                 [FKM_GENEVE_BIT] = "geneve",
1430                 [FKM_NSH_BIT] = "nsh",
1431                 [FKM_ROCEV2_BIT] = "rocev2",
1432                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1433                 [FKM_IPOK_BIT] = "ipok",
1434                 [FKM_L4OK_BIT] = "l4ok",
1435                 [FKM_ROCEOK_BIT] = "roceok",
1436                 [FKM_FCSOK_BIT] = "fcsok",
1437                 [FKM_EG_SPAN_BIT] = "eg_span",
1438                 [FKM_IG_SPAN_BIT] = "ig_span",
1439                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1440         };
1441         /* Manually keep in sync with FKH_BITS */
1442         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1443                 [FKH_ETHER_BIT] = "eth",
1444                 [FKH_QTAG_BIT] = "qtag",
1445                 [FKH_L2RAW_BIT] = "l2raw",
1446                 [FKH_IPV4_BIT] = "ip4",
1447                 [FKH_IPV6_BIT] = "ip6",
1448                 [FKH_L3RAW_BIT] = "l3raw",
1449                 [FKH_UDP_BIT] = "udp",
1450                 [FKH_TCP_BIT] = "tcp",
1451                 [FKH_ICMP_BIT] = "icmp",
1452                 [FKH_VXLAN_BIT] = "vxlan",
1453                 [FKH_L4RAW_BIT] = "l4raw",
1454         };
1455         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1456         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1457         int n;
1458
1459         if (!fkm_bits && !fkh_bits)
1460                 return;
1461         n = snprintf(buf, buf_len, "metadata(");
1462         if (n > 0 && n < buf_len) {
1463                 buf += n;
1464                 buf_len -= n;
1465         }
1466         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1467         if (n > 0 && n < buf_len) {
1468                 buf += n;
1469                 buf_len -= n;
1470         }
1471         n = snprintf(buf, buf_len, ") valid hdr fields(");
1472         if (n > 0 && n < buf_len) {
1473                 buf += n;
1474                 buf_len -= n;
1475         }
1476         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1477         if (n > 0 && n < buf_len) {
1478                 buf += n;
1479                 buf_len -= n;
1480         }
1481         snprintf(buf, buf_len, ")");
1482 }
1483
1484 static void
1485 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1486                         uint8_t ingress)
1487 {
1488         char buf[256];
1489
1490         memset(buf, 0, sizeof(buf));
1491         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1492                                   buf, sizeof(buf));
1493         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1494                     (ingress) ? "IG" : "EG", buf,
1495                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1496         memset(buf, 0, sizeof(buf));
1497         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1498                                   buf, sizeof(buf));
1499         if (buf[0])
1500                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1501 }
1502
1503 /* Debug function to dump internal NIC flow structures. */
1504 static void
1505 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1506                         const struct fm_action *fm_action,
1507                         uint8_t ingress)
1508 {
1509         if (rte_log_get_level(enic_pmd_logtype) < (int)RTE_LOG_DEBUG)
1510                 return;
1511         enic_fm_dump_tcam_match(fm_match, ingress);
1512         enic_fm_dump_tcam_actions(fm_action);
1513 }
1514
1515 static int
1516 enic_fm_flow_parse(struct enic_flowman *fm,
1517                    const struct rte_flow_attr *attrs,
1518                    const struct rte_flow_item pattern[],
1519                    const struct rte_flow_action actions[],
1520                    struct rte_flow_error *error)
1521 {
1522         const struct rte_flow_action *action;
1523         unsigned int ret;
1524         static const enum rte_flow_action_type *sa;
1525
1526         ENICPMD_FUNC_TRACE();
1527         ret = 0;
1528         if (!pattern) {
1529                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1530                                    NULL, "no pattern specified");
1531                 return -rte_errno;
1532         }
1533
1534         if (!actions) {
1535                 rte_flow_error_set(error, EINVAL,
1536                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1537                                    NULL, "no action specified");
1538                 return -rte_errno;
1539         }
1540
1541         if (attrs) {
1542                 if (attrs->priority) {
1543                         rte_flow_error_set(error, ENOTSUP,
1544                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1545                                            NULL,
1546                                            "priorities are not supported");
1547                         return -rte_errno;
1548                 } else if (attrs->transfer) {
1549                         rte_flow_error_set(error, ENOTSUP,
1550                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1551                                            NULL,
1552                                            "transfer is not supported");
1553                         return -rte_errno;
1554                 } else if (attrs->ingress && attrs->egress) {
1555                         rte_flow_error_set(error, ENOTSUP,
1556                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1557                                            NULL,
1558                                            "bidirectional rules not supported");
1559                         return -rte_errno;
1560                 }
1561
1562         } else {
1563                 rte_flow_error_set(error, EINVAL,
1564                                    RTE_FLOW_ERROR_TYPE_ATTR,
1565                                    NULL, "no attribute specified");
1566                 return -rte_errno;
1567         }
1568
1569         /* Verify Actions. */
1570         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1571              enic_fm_supported_eg_actions;
1572         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1573              action++) {
1574                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1575                         continue;
1576                 else if (!enic_fm_match_action(action, sa))
1577                         break;
1578         }
1579         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1580                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1581                                    action, "invalid action");
1582                 return -rte_errno;
1583         }
1584         ret = enic_fm_copy_entry(fm, pattern, error);
1585         if (ret)
1586                 return ret;
1587         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1588         return ret;
1589 }
1590
1591 static void
1592 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1593 {
1594         if (!fm_flow->counter_valid)
1595                 return;
1596         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1597         fm_flow->counter_valid = false;
1598 }
1599
1600 static int
1601 enic_fm_more_counters(struct enic_flowman *fm)
1602 {
1603         struct enic_fm_counter *new_stack;
1604         struct enic_fm_counter *ctrs;
1605         struct enic *enic;
1606         int i, rc;
1607         u64 args[2];
1608
1609         ENICPMD_FUNC_TRACE();
1610         enic = fm->enic;
1611         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1612                                 FM_COUNTERS_EXPAND) *
1613                                 sizeof(struct enic_fm_counter), 0);
1614         if (new_stack == NULL) {
1615                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1616                 return -ENOMEM;
1617         }
1618         fm->counter_stack = new_stack;
1619
1620         args[0] = FM_COUNTER_BRK;
1621         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1622         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1623         if (rc != 0) {
1624                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1625                 return rc;
1626         }
1627         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1628                 fm->counters_alloced;
1629         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1630                 ctrs->handle = fm->counters_alloced + i;
1631                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1632         }
1633         fm->counters_alloced += FM_COUNTERS_EXPAND;
1634         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1635                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1636         return 0;
1637 }
1638
1639 static int
1640 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1641 {
1642         struct enic *enic;
1643         u64 args[3];
1644         int ret;
1645
1646         ENICPMD_FUNC_TRACE();
1647         enic = fm->enic;
1648         args[0] = FM_COUNTER_QUERY;
1649         args[1] = c->handle;
1650         args[2] = 1; /* clear */
1651         ret = vnic_dev_flowman_cmd(enic->vdev, args, 3);
1652         if (ret) {
1653                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1654                             ret, c->handle);
1655                 return ret;
1656         }
1657         return 0;
1658 }
1659
1660 static int
1661 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1662                       struct enic_fm_counter **ctr)
1663 {
1664         struct enic_fm_counter *c;
1665         int ret;
1666
1667         ENICPMD_FUNC_TRACE();
1668         *ctr = NULL;
1669         if (SLIST_EMPTY(&fm->counters)) {
1670                 ret = enic_fm_more_counters(fm);
1671                 if (ret)
1672                         return rte_flow_error_set(error, -ret,
1673                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1674                                 NULL, "enic: out of counters");
1675         }
1676         c = SLIST_FIRST(&fm->counters);
1677         SLIST_REMOVE_HEAD(&fm->counters, next);
1678         *ctr = c;
1679         return 0;
1680 }
1681
1682 static int
1683 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1684 {
1685         u64 args[2];
1686         int rc;
1687
1688         ENICPMD_FUNC_TRACE();
1689         args[0] = FM_ACTION_FREE;
1690         args[1] = handle;
1691         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1692         if (rc)
1693                 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1694                             rc, handle);
1695         return rc;
1696 }
1697
1698 static int
1699 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1700 {
1701         u64 args[2];
1702         int rc;
1703
1704         ENICPMD_FUNC_TRACE();
1705         args[0] = FM_MATCH_ENTRY_REMOVE;
1706         args[1] = handle;
1707         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1708         if (rc)
1709                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1710                             " handle=0x%" PRIx64, rc, handle);
1711         return rc;
1712 }
1713
1714 static struct enic_fm_jump_flow *
1715 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1716 {
1717         struct enic_fm_jump_flow *j;
1718
1719         ENICPMD_FUNC_TRACE();
1720         TAILQ_FOREACH(j, &fm->jump_list, list) {
1721                 if (j->group == group)
1722                         return j;
1723         }
1724         return NULL;
1725 }
1726
1727 static void
1728 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1729 {
1730         struct enic_fm_jump_flow *j;
1731
1732         ENICPMD_FUNC_TRACE();
1733         TAILQ_FOREACH(j, &fm->jump_list, list) {
1734                 if (j->flow == flow) {
1735                         TAILQ_REMOVE(&fm->jump_list, j, list);
1736                         free(j);
1737                         return;
1738                 }
1739         }
1740 }
1741
1742 static int
1743 save_jump_flow(struct enic_flowman *fm,
1744                struct rte_flow *flow,
1745                uint32_t group,
1746                struct fm_tcam_match_entry *match,
1747                struct fm_action *action)
1748 {
1749         struct enic_fm_jump_flow *j;
1750
1751         ENICPMD_FUNC_TRACE();
1752         j = calloc(1, sizeof(struct enic_fm_jump_flow));
1753         if (j == NULL)
1754                 return -ENOMEM;
1755         j->flow = flow;
1756         j->group = group;
1757         j->match = *match;
1758         j->action = *action;
1759         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1760         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1761         return 0;
1762 }
1763
1764 static void
1765 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1766 {
1767         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1768                 enic_fm_entry_free(fm, fm_flow->entry_handle);
1769                 fm_flow->entry_handle = FM_INVALID_HANDLE;
1770         }
1771         if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1772                 enic_fm_action_free(fm, fm_flow->action_handle);
1773                 fm_flow->action_handle = FM_INVALID_HANDLE;
1774         }
1775         enic_fm_counter_free(fm, fm_flow);
1776         if (fm_flow->fet) {
1777                 enic_fet_put(fm, fm_flow->fet);
1778                 fm_flow->fet = NULL;
1779         }
1780 }
1781
1782 static void
1783 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1784 {
1785         if (flow->fm->fet && flow->fm->fet->default_key)
1786                 remove_jump_flow(fm, flow);
1787         __enic_fm_flow_free(fm, flow->fm);
1788         free(flow->fm);
1789         free(flow);
1790 }
1791
1792 static int
1793 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1794                        struct fm_tcam_match_entry *match_in,
1795                        uint64_t *entry_handle,
1796                        uint8_t ingress,
1797                        struct rte_flow_error *error)
1798 {
1799         struct fm_tcam_match_entry *ftm;
1800         u64 args[3];
1801         int ret;
1802
1803         ENICPMD_FUNC_TRACE();
1804         /* Copy entry to the command buffer */
1805         ftm = &fm->cmd.va->fm_tcam_match_entry;
1806         memcpy(ftm, match_in, sizeof(*ftm));
1807         /* Add TCAM entry */
1808         args[0] = FM_TCAM_ENTRY_INSTALL;
1809         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1810         args[2] = fm->cmd.pa;
1811         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1812         if (ret != 0) {
1813                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1814                             ingress ? "ingress" : "egress", ret);
1815                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1816                         NULL, "enic: devcmd(tcam-entry-install)");
1817                 return ret;
1818         }
1819         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1820                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
1821         *entry_handle = args[0];
1822         return 0;
1823 }
1824
1825 static int
1826 enic_fm_add_exact_entry(struct enic_flowman *fm,
1827                         struct fm_tcam_match_entry *match_in,
1828                         uint64_t *entry_handle,
1829                         struct enic_fm_fet *fet,
1830                         struct rte_flow_error *error)
1831 {
1832         struct fm_exact_match_entry *fem;
1833         u64 args[3];
1834         int ret;
1835
1836         ENICPMD_FUNC_TRACE();
1837         /* The new entry must have the table's key */
1838         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1839                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1840                 return rte_flow_error_set(error, EINVAL,
1841                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1842                         "enic: key does not match group's key");
1843         }
1844
1845         /* Copy entry to the command buffer */
1846         fem = &fm->cmd.va->fm_exact_match_entry;
1847         /*
1848          * Translate TCAM entry to exact entry. As is only need to drop
1849          * position and mask. The mask is part of the exact match table.
1850          * Position (aka priority) is not supported in the exact match table.
1851          */
1852         fem->fem_data = match_in->ftm_data;
1853         fem->fem_flags = match_in->ftm_flags;
1854         fem->fem_action = match_in->ftm_action;
1855         fem->fem_counter = match_in->ftm_counter;
1856
1857         /* Add exact entry */
1858         args[0] = FM_EXACT_ENTRY_INSTALL;
1859         args[1] = fet->handle;
1860         args[2] = fm->cmd.pa;
1861         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1862         if (ret != 0) {
1863                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
1864                             fet->ingress ? "ingress" : "egress", fet->group);
1865                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1866                         NULL, "enic: devcmd(exact-entry-install)");
1867                 return ret;
1868         }
1869         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
1870                     " handle=0x%" PRIx64,
1871                     fet->ingress ? "ingress" : "egress", fet->group,
1872                     (uint64_t)args[0]);
1873         *entry_handle = args[0];
1874         return 0;
1875 }
1876
1877 /* Push match-action to the NIC. */
1878 static int
1879 __enic_fm_flow_add_entry(struct enic_flowman *fm,
1880                          struct enic_fm_flow *fm_flow,
1881                          struct fm_tcam_match_entry *match_in,
1882                          struct fm_action *action_in,
1883                          uint32_t group,
1884                          uint8_t ingress,
1885                          struct rte_flow_error *error)
1886 {
1887         struct enic_fm_counter *ctr;
1888         struct fm_action *fma;
1889         uint64_t action_h;
1890         uint64_t entry_h;
1891         u64 args[3];
1892         int ret;
1893
1894         ENICPMD_FUNC_TRACE();
1895         /* Allocate action. */
1896         fma = &fm->cmd.va->fm_action;
1897         memcpy(fma, action_in, sizeof(*fma));
1898         args[0] = FM_ACTION_ALLOC;
1899         args[1] = fm->cmd.pa;
1900         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1901         if (ret != 0) {
1902                 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
1903                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1904                         NULL, "enic: devcmd(action-alloc)");
1905                 return ret;
1906         }
1907         action_h = args[0];
1908         fm_flow->action_handle = action_h;
1909         match_in->ftm_action = action_h;
1910         ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
1911
1912         /* Allocate counter if requested. */
1913         if (match_in->ftm_flags & FMEF_COUNTER) {
1914                 ret = enic_fm_counter_alloc(fm, error, &ctr);
1915                 if (ret) /* error has been filled in */
1916                         return ret;
1917                 fm_flow->counter_valid = true;
1918                 fm_flow->counter = ctr;
1919                 match_in->ftm_counter = ctr->handle;
1920         }
1921
1922         /*
1923          * Get the group's table (either TCAM or exact match table) and
1924          * add entry to it. If we use the exact match table, the handler
1925          * will translate the TCAM entry (match_in) to the appropriate
1926          * exact match entry and use that instead.
1927          */
1928         entry_h = FM_INVALID_HANDLE;
1929         if (group == FM_TCAM_RTE_GROUP) {
1930                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
1931                                              error);
1932                 if (ret)
1933                         return ret;
1934                 /* Jump action might have a ref to fet */
1935                 fm_flow->fet = fm->fet;
1936                 fm->fet = NULL;
1937         } else {
1938                 struct enic_fm_fet *fet = NULL;
1939
1940                 ret = enic_fet_get(fm, group, ingress,
1941                                    &match_in->ftm_mask, &fet, error);
1942                 if (ret)
1943                         return ret;
1944                 fm_flow->fet = fet;
1945                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
1946                                               error);
1947                 if (ret)
1948                         return ret;
1949         }
1950         /* Clear counter after adding entry, as it requires in-use counter */
1951         if (fm_flow->counter_valid) {
1952                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
1953                 if (ret)
1954                         return ret;
1955         }
1956         fm_flow->entry_handle = entry_h;
1957         return 0;
1958 }
1959
1960 /* Push match-action to the NIC. */
1961 static struct rte_flow *
1962 enic_fm_flow_add_entry(struct enic_flowman *fm,
1963                        struct fm_tcam_match_entry *match_in,
1964                        struct fm_action *action_in,
1965                        const struct rte_flow_attr *attrs,
1966                        struct rte_flow_error *error)
1967 {
1968         struct enic_fm_flow *fm_flow;
1969         struct rte_flow *flow;
1970
1971         ENICPMD_FUNC_TRACE();
1972         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
1973         flow = calloc(1, sizeof(*flow));
1974         fm_flow = calloc(1, sizeof(*fm_flow));
1975         if (flow == NULL || fm_flow == NULL) {
1976                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1977                         NULL, "enic: cannot allocate rte_flow");
1978                 free(flow);
1979                 free(fm_flow);
1980                 return NULL;
1981         }
1982         flow->fm = fm_flow;
1983         fm_flow->action_handle = FM_INVALID_HANDLE;
1984         fm_flow->entry_handle = FM_INVALID_HANDLE;
1985         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
1986                                      attrs->group, attrs->ingress, error)) {
1987                 enic_fm_flow_free(fm, flow);
1988                 return NULL;
1989         }
1990         return flow;
1991 }
1992
1993 static void
1994 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
1995                    struct rte_flow_error *error)
1996 {
1997         struct enic_fm_flow *fm_flow;
1998         struct enic_fm_jump_flow *j;
1999         struct fm_action *fma;
2000         uint32_t group;
2001
2002         ENICPMD_FUNC_TRACE();
2003         /*
2004          * Find the saved flows that should jump to the new table (fet).
2005          * Then delete the old TCAM entry that jumps to the default table,
2006          * and add a new one that jumps to the new table.
2007          */
2008         group = fet->group;
2009         j = find_jump_flow(fm, group);
2010         while (j) {
2011                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2012                             j->flow, group);
2013                 /* Delete old entry */
2014                 fm_flow = j->flow->fm;
2015                 __enic_fm_flow_free(fm, fm_flow);
2016
2017                 /* Add new entry */
2018                 fma = &j->action;
2019                 fma->fma_action_ops[0].exact.handle = fet->handle;
2020                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2021                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2022                         /* Cannot roll back changes at the moment */
2023                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2024                                     j->flow);
2025                 } else {
2026                         fm_flow->fet = fet;
2027                         fet->ref++;
2028                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2029                                     fet->group, fet->ref);
2030                 }
2031
2032                 TAILQ_REMOVE(&fm->jump_list, j, list);
2033                 free(j);
2034                 j = find_jump_flow(fm, group);
2035         }
2036 }
2037
2038 static void
2039 enic_fm_open_scratch(struct enic_flowman *fm)
2040 {
2041         fm->action_op_count = 0;
2042         fm->fet = NULL;
2043         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2044         memset(&fm->action, 0, sizeof(fm->action));
2045 }
2046
2047 static void
2048 enic_fm_close_scratch(struct enic_flowman *fm)
2049 {
2050         if (fm->fet) {
2051                 enic_fet_put(fm, fm->fet);
2052                 fm->fet = NULL;
2053         }
2054         fm->action_op_count = 0;
2055 }
2056
2057 static int
2058 enic_fm_flow_validate(struct rte_eth_dev *dev,
2059                       const struct rte_flow_attr *attrs,
2060                       const struct rte_flow_item pattern[],
2061                       const struct rte_flow_action actions[],
2062                       struct rte_flow_error *error)
2063 {
2064         struct fm_tcam_match_entry *fm_tcam_entry;
2065         struct fm_action *fm_action;
2066         struct enic_flowman *fm;
2067         int ret;
2068
2069         ENICPMD_FUNC_TRACE();
2070         fm = pmd_priv(dev)->fm;
2071         if (fm == NULL)
2072                 return -ENOTSUP;
2073         enic_fm_open_scratch(fm);
2074         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2075         if (!ret) {
2076                 fm_tcam_entry = &fm->tcam_entry;
2077                 fm_action = &fm->action;
2078                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2079                                         attrs->ingress);
2080         }
2081         enic_fm_close_scratch(fm);
2082         return ret;
2083 }
2084
2085 static int
2086 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2087                          struct rte_flow *flow, void *data,
2088                          struct rte_flow_error *error)
2089 {
2090         struct rte_flow_query_count *query;
2091         struct enic_fm_flow *fm_flow;
2092         struct enic *enic;
2093         u64 args[3];
2094         int rc;
2095
2096         ENICPMD_FUNC_TRACE();
2097         enic = pmd_priv(dev);
2098         query = data;
2099         fm_flow = flow->fm;
2100         if (!fm_flow->counter_valid)
2101                 return rte_flow_error_set(error, ENOTSUP,
2102                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2103                         "enic: flow does not have counter");
2104
2105         args[0] = FM_COUNTER_QUERY;
2106         args[1] = fm_flow->counter->handle;
2107         args[2] = query->reset;
2108         rc = vnic_dev_flowman_cmd(enic->vdev, args, 3);
2109         if (rc) {
2110                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2111                             rc, fm_flow->counter->handle);
2112                 return rc;
2113         }
2114         query->hits_set = 1;
2115         query->hits = args[0];
2116         query->bytes_set = 1;
2117         query->bytes = args[1];
2118         return 0;
2119 }
2120
2121 static int
2122 enic_fm_flow_query(struct rte_eth_dev *dev,
2123                    struct rte_flow *flow,
2124                    const struct rte_flow_action *actions,
2125                    void *data,
2126                    struct rte_flow_error *error)
2127 {
2128         int ret = 0;
2129
2130         ENICPMD_FUNC_TRACE();
2131         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2132                 switch (actions->type) {
2133                 case RTE_FLOW_ACTION_TYPE_VOID:
2134                         break;
2135                 case RTE_FLOW_ACTION_TYPE_COUNT:
2136                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2137                         break;
2138                 default:
2139                         return rte_flow_error_set(error, ENOTSUP,
2140                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2141                                                   actions,
2142                                                   "action not supported");
2143                 }
2144                 if (ret < 0)
2145                         return ret;
2146         }
2147         return 0;
2148 }
2149
2150 static struct rte_flow *
2151 enic_fm_flow_create(struct rte_eth_dev *dev,
2152                     const struct rte_flow_attr *attrs,
2153                     const struct rte_flow_item pattern[],
2154                     const struct rte_flow_action actions[],
2155                     struct rte_flow_error *error)
2156 {
2157         struct fm_tcam_match_entry *fm_tcam_entry;
2158         struct fm_action *fm_action;
2159         struct enic_flowman *fm;
2160         struct enic_fm_fet *fet;
2161         struct rte_flow *flow;
2162         struct enic *enic;
2163         int ret;
2164
2165         ENICPMD_FUNC_TRACE();
2166         enic = pmd_priv(dev);
2167         fm = enic->fm;
2168         if (fm == NULL) {
2169                 rte_flow_error_set(error, ENOTSUP,
2170                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2171                         "flowman is not initialized");
2172                 return NULL;
2173         }
2174         enic_fm_open_scratch(fm);
2175         flow = NULL;
2176         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2177         if (ret < 0)
2178                 goto error_with_scratch;
2179         fm_tcam_entry = &fm->tcam_entry;
2180         fm_action = &fm->action;
2181         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2182                                       attrs, error);
2183         if (flow) {
2184                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2185                 fet = flow->fm->fet;
2186                 if (fet && fet->default_key) {
2187                         /*
2188                          * Jump to non-existent group? Save the relevant info
2189                          * so we can convert this flow when that group
2190                          * materializes.
2191                          */
2192                         save_jump_flow(fm, flow, fet->group,
2193                                        fm_tcam_entry, fm_action);
2194                 } else if (fet && fet->ref == 1) {
2195                         /*
2196                          * A new table is created. Convert the saved flows
2197                          * that should jump to this group.
2198                          */
2199                         convert_jump_flows(fm, fet, error);
2200                 }
2201         }
2202
2203 error_with_scratch:
2204         enic_fm_close_scratch(fm);
2205         return flow;
2206 }
2207
2208 static int
2209 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2210                      __rte_unused struct rte_flow_error *error)
2211 {
2212         struct enic *enic = pmd_priv(dev);
2213
2214         ENICPMD_FUNC_TRACE();
2215         if (enic->fm == NULL)
2216                 return 0;
2217         LIST_REMOVE(flow, next);
2218         enic_fm_flow_free(enic->fm, flow);
2219         return 0;
2220 }
2221
2222 static int
2223 enic_fm_flow_flush(struct rte_eth_dev *dev,
2224                    __rte_unused struct rte_flow_error *error)
2225 {
2226         struct enic_fm_flow *fm_flow;
2227         struct enic_flowman *fm;
2228         struct rte_flow *flow;
2229         struct enic *enic = pmd_priv(dev);
2230
2231         ENICPMD_FUNC_TRACE();
2232         if (enic->fm == NULL)
2233                 return 0;
2234         fm = enic->fm;
2235         while (!LIST_EMPTY(&enic->flows)) {
2236                 flow = LIST_FIRST(&enic->flows);
2237                 fm_flow = flow->fm;
2238                 LIST_REMOVE(flow, next);
2239                 /*
2240                  * If tables are null, then vNIC is closing, and the firmware
2241                  * has already cleaned up flowman state. So do not try to free
2242                  * resources, as it only causes errors.
2243                  */
2244                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2245                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2246                         fm_flow->action_handle = FM_INVALID_HANDLE;
2247                         fm_flow->fet = NULL;
2248                 }
2249                 enic_fm_flow_free(fm, flow);
2250         }
2251         return 0;
2252 }
2253
2254 static int
2255 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2256 {
2257         u64 args[2];
2258         int rc;
2259
2260         args[0] = FM_MATCH_TABLE_FREE;
2261         args[1] = handle;
2262         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
2263         if (rc)
2264                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2265                             rc, handle);
2266         return rc;
2267 }
2268
2269 static int
2270 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2271                         uint32_t max_entries, uint64_t *handle)
2272 {
2273         struct fm_tcam_match_table *tcam_tbl;
2274         struct enic *enic;
2275         u64 args[2];
2276         int rc;
2277
2278         ENICPMD_FUNC_TRACE();
2279         enic = fm->enic;
2280         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2281         tcam_tbl->ftt_direction = direction;
2282         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2283         tcam_tbl->ftt_max_entries = max_entries;
2284         args[0] = FM_TCAM_TABLE_ALLOC;
2285         args[1] = fm->cmd.pa;
2286         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2287         if (rc) {
2288                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2289                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2290                 return rc;
2291         }
2292         *handle = args[0];
2293         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2294                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2295         return 0;
2296 }
2297
2298 static int
2299 enic_fm_init_counters(struct enic_flowman *fm)
2300 {
2301         ENICPMD_FUNC_TRACE();
2302         SLIST_INIT(&fm->counters);
2303         return enic_fm_more_counters(fm);
2304 }
2305
2306 static void
2307 enic_fm_free_all_counters(struct enic_flowman *fm)
2308 {
2309         struct enic *enic;
2310         u64 args[2];
2311         int rc;
2312
2313         enic = fm->enic;
2314         args[0] = FM_COUNTER_BRK;
2315         args[1] = 0;
2316         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2317         if (rc != 0)
2318                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2319         rte_free(fm->counter_stack);
2320 }
2321
2322 static int
2323 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2324 {
2325         int rc;
2326
2327         ENICPMD_FUNC_TRACE();
2328         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2329                                     &fm->ig_tcam_hndl);
2330         if (rc)
2331                 return rc;
2332         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2333                                     &fm->eg_tcam_hndl);
2334         return rc;
2335 }
2336
2337 static void
2338 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2339 {
2340         ENICPMD_FUNC_TRACE();
2341         if (fm->ig_tcam_hndl) {
2342                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2343                             fm->ig_tcam_hndl);
2344                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2345                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2346         }
2347         if (fm->eg_tcam_hndl) {
2348                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2349                             fm->eg_tcam_hndl);
2350                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2351                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2352         }
2353 }
2354
2355 int
2356 enic_fm_init(struct enic *enic)
2357 {
2358         struct enic_flowman *fm;
2359         u8 name[NAME_MAX];
2360         int rc;
2361
2362         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2363                 return 0;
2364         ENICPMD_FUNC_TRACE();
2365         fm = calloc(1, sizeof(*fm));
2366         if (fm == NULL) {
2367                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2368                 return -ENOMEM;
2369         }
2370         fm->enic = enic;
2371         TAILQ_INIT(&fm->fet_list);
2372         TAILQ_INIT(&fm->jump_list);
2373         /* Allocate host memory for flowman commands */
2374         snprintf((char *)name, NAME_MAX, "fm-cmd-%s", enic->bdf_name);
2375         fm->cmd.va = enic_alloc_consistent(enic,
2376                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2377         if (!fm->cmd.va) {
2378                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2379                 rc = -ENOMEM;
2380                 goto error_fm;
2381         }
2382         /* Allocate TCAM tables upfront as they are the main tables */
2383         rc = enic_fm_alloc_tcam_tables(fm);
2384         if (rc) {
2385                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2386                 goto error_cmd;
2387         }
2388         /* Then a number of counters */
2389         rc = enic_fm_init_counters(fm);
2390         if (rc) {
2391                 ENICPMD_LOG(ERR, "cannot alloc counters");
2392                 goto error_tables;
2393         }
2394         /*
2395          * One default exact match table for each direction. We hold onto
2396          * it until close.
2397          */
2398         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2399         if (rc) {
2400                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2401                 goto error_counters;
2402         }
2403         fm->default_ig_fet->ref = 1;
2404         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2405         if (rc) {
2406                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2407                 goto error_ig_fet;
2408         }
2409         fm->default_eg_fet->ref = 1;
2410         enic->fm = fm;
2411         return 0;
2412
2413 error_ig_fet:
2414         enic_fet_free(fm, fm->default_ig_fet);
2415 error_counters:
2416         enic_fm_free_all_counters(fm);
2417 error_tables:
2418         enic_fm_free_tcam_tables(fm);
2419 error_cmd:
2420         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2421                 fm->cmd.va, fm->cmd.pa);
2422 error_fm:
2423         free(fm);
2424         return rc;
2425 }
2426
2427 void
2428 enic_fm_destroy(struct enic *enic)
2429 {
2430         struct enic_flowman *fm;
2431         struct enic_fm_fet *fet;
2432
2433         if (enic->fm == NULL)
2434                 return;
2435         ENICPMD_FUNC_TRACE();
2436         fm = enic->fm;
2437         enic_fet_free(fm, fm->default_eg_fet);
2438         enic_fet_free(fm, fm->default_ig_fet);
2439         /* Free all exact match tables still open */
2440         while (!TAILQ_EMPTY(&fm->fet_list)) {
2441                 fet = TAILQ_FIRST(&fm->fet_list);
2442                 enic_fet_free(fm, fet);
2443         }
2444         enic_fm_free_tcam_tables(fm);
2445         enic_fm_free_all_counters(fm);
2446         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2447                 fm->cmd.va, fm->cmd.pa);
2448         fm->cmd.va = NULL;
2449         free(fm);
2450         enic->fm = NULL;
2451 }
2452
2453 const struct rte_flow_ops enic_fm_flow_ops = {
2454         .validate = enic_fm_flow_validate,
2455         .create = enic_fm_flow_create,
2456         .destroy = enic_fm_flow_destroy,
2457         .flush = enic_fm_flow_flush,
2458         .query = enic_fm_flow_query,
2459 };