drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_ip.h>
  12 #include <rte_udp.h>
  13
  14 #include "enic_compat.h"
  15 #include "enic.h"
  16 #include "vnic_dev.h"
  17 #include "vnic_nic.h"
  18
  19 #define IP_DEFTTL  64   /* from RFC 1340. */
  20 #define IP6_VTC_FLOW 0x60000000
  21
  22 /* Highest Item type supported by Flowman */
  23 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  24
  25 /* Up to 1024 TCAM entries */
  26 #define FM_MAX_TCAM_TABLE_SIZE 1024
  27
  28 /* Up to 4096 entries per exact match table */
  29 #define FM_MAX_EXACT_TABLE_SIZE 4096
  30
  31 /* Number of counters to increase on for each increment */
  32 #define FM_COUNTERS_EXPAND  100
  33
  34 #define FM_INVALID_HANDLE 0
  35
  36 /*
  37  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  38  * Use a simple scheme to map groups to tables.
  39  * Group 0 uses the single TCAM tables, one for each direction.
  40  * Group 1, 2, ... uses its own exact match table.
  41  *
  42  * The TCAM tables are allocated upfront during init.
  43  *
  44  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  45  *
  46  * 1. Add a flow that jumps from group 0 to group N.
  47  *
  48  * If N does not exist, we allocate an exact match table for it, using
  49  * a dummy key. A key is required for the table.
  50  *
  51  * 2. Add a flow that uses group N.
  52  *
  53  * If N does not exist, we allocate an exact match table for it, using
  54  * the flow's key. Subsequent flows to the same group all should have
  55  * the same key.
  56  *
  57  * Without a jump flow to N, N is not reachable in hardware. No packets
  58  * reach N and match.
  59  *
  60  * 3. Add a flow to an empty group N.
  61  *
  62  * N has been created via (1) and the dummy key. We free that table, allocate
  63  * a new table using the new flow's key. Also re-do the existing jump flow to
  64  * point to the new table.
  65  */
  66 #define FM_TCAM_RTE_GROUP 0
  67
  68 struct enic_fm_fet {
  69         TAILQ_ENTRY(enic_fm_fet) list;
  70         uint32_t group; /* rte_flow group ID */
  71         uint64_t handle; /* Exact match table handle from flowman */
  72         uint8_t ingress;
  73         uint8_t default_key;
  74         int ref; /* Reference count via get/put */
  75         struct fm_key_template key; /* Key associated with the table */
  76 };
  77
  78 struct enic_fm_counter {
  79         SLIST_ENTRY(enic_fm_counter) next;
  80         uint32_t handle;
  81 };
  82
  83 /* rte_flow.fm */
  84 struct enic_fm_flow {
  85         bool counter_valid;
  86         uint64_t entry_handle;
  87         uint64_t action_handle;
  88         struct enic_fm_counter *counter;
  89         struct enic_fm_fet *fet;
  90 };
  91
  92 struct enic_fm_jump_flow {
  93         TAILQ_ENTRY(enic_fm_jump_flow) list;
  94         struct rte_flow *flow;
  95         uint32_t group;
  96         struct fm_tcam_match_entry match;
  97         struct fm_action action;
  98 };
  99
 100 /*
 101  * Flowman uses host memory for commands. This structure is allocated
 102  * in DMA-able memory.
 103  */
 104 union enic_flowman_cmd_mem {
 105         struct fm_tcam_match_table fm_tcam_match_table;
 106         struct fm_exact_match_table fm_exact_match_table;
 107         struct fm_tcam_match_entry fm_tcam_match_entry;
 108         struct fm_exact_match_entry fm_exact_match_entry;
 109         struct fm_action fm_action;
 110 };
 111
 112 struct enic_flowman {
 113         struct enic *enic;
 114         /* Command buffer */
 115         struct {
 116                 union enic_flowman_cmd_mem *va;
 117                 dma_addr_t pa;
 118         } cmd;
 119         /* TCAM tables allocated upfront, used for group 0 */
 120         uint64_t ig_tcam_hndl;
 121         uint64_t eg_tcam_hndl;
 122         /* Counters */
 123         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 124         void *counter_stack;
 125         uint32_t counters_alloced;
 126         /* Exact match tables for groups != 0, dynamically allocated */
 127         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 128         /*
 129          * Default exact match tables used for jump actions to
 130          * non-existent groups.
 131          */
 132         struct enic_fm_fet *default_eg_fet;
 133         struct enic_fm_fet *default_ig_fet;
 134         /* Flows that jump to the default table above */
 135         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 136         /*
 137          * Scratch data used during each invocation of flow_create
 138          * and flow_validate.
 139          */
 140         struct enic_fm_fet *fet;
 141         struct fm_tcam_match_entry tcam_entry;
 142         struct fm_action action;
 143         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 144         int action_op_count;
 145 };
 146
 147 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 148
 149 /*
 150  * Common arguments passed to copy_item functions. Use this structure
 151  * so we can easily add new arguments.
 152  * item: Item specification.
 153  * fm_tcam_entry: Flowman TCAM match entry.
 154  * header_level: 0 for outer header, 1 for inner header.
 155  */
 156 struct copy_item_args {
 157         const struct rte_flow_item *item;
 158         struct fm_tcam_match_entry *fm_tcam_entry;
 159         uint8_t header_level;
 160 };
 161
 162 /* functions for copying items into flowman match */
 163 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 164
 165 /* Info about how to copy items into flowman match */
 166 struct enic_fm_items {
 167         /* Function for copying and validating an item. */
 168         enic_copy_item_fn * const copy_item;
 169         /* List of valid previous items. */
 170         const enum rte_flow_item_type * const prev_items;
 171         /*
 172          * True if it's OK for this item to be the first item. For some NIC
 173          * versions, it's invalid to start the stack above layer 3.
 174          */
 175         const uint8_t valid_start_item;
 176 };
 177
 178 static enic_copy_item_fn enic_fm_copy_item_eth;
 179 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 180 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 181 static enic_copy_item_fn enic_fm_copy_item_raw;
 182 static enic_copy_item_fn enic_fm_copy_item_sctp;
 183 static enic_copy_item_fn enic_fm_copy_item_tcp;
 184 static enic_copy_item_fn enic_fm_copy_item_udp;
 185 static enic_copy_item_fn enic_fm_copy_item_vlan;
 186 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 187
 188 /* Ingress actions */
 189 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 190         RTE_FLOW_ACTION_TYPE_COUNT,
 191         RTE_FLOW_ACTION_TYPE_DROP,
 192         RTE_FLOW_ACTION_TYPE_FLAG,
 193         RTE_FLOW_ACTION_TYPE_JUMP,
 194         RTE_FLOW_ACTION_TYPE_MARK,
 195         RTE_FLOW_ACTION_TYPE_PORT_ID,
 196         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 197         RTE_FLOW_ACTION_TYPE_QUEUE,
 198         RTE_FLOW_ACTION_TYPE_RSS,
 199         RTE_FLOW_ACTION_TYPE_VOID,
 200         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 201         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 202         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 203 };
 204
 205 /* Egress actions */
 206 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 207         RTE_FLOW_ACTION_TYPE_COUNT,
 208         RTE_FLOW_ACTION_TYPE_DROP,
 209         RTE_FLOW_ACTION_TYPE_JUMP,
 210         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 211         RTE_FLOW_ACTION_TYPE_VOID,
 212         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 213         RTE_FLOW_ACTION_TYPE_END,
 214 };
 215
 216 static const struct enic_fm_items enic_fm_items[] = {
 217         [RTE_FLOW_ITEM_TYPE_RAW] = {
 218                 .copy_item = enic_fm_copy_item_raw,
 219                 .valid_start_item = 0,
 220                 .prev_items = (const enum rte_flow_item_type[]) {
 221                                RTE_FLOW_ITEM_TYPE_UDP,
 222                                RTE_FLOW_ITEM_TYPE_END,
 223                 },
 224         },
 225         [RTE_FLOW_ITEM_TYPE_ETH] = {
 226                 .copy_item = enic_fm_copy_item_eth,
 227                 .valid_start_item = 1,
 228                 .prev_items = (const enum rte_flow_item_type[]) {
 229                                RTE_FLOW_ITEM_TYPE_END,
 230                 },
 231         },
 232         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 233                 .copy_item = enic_fm_copy_item_vlan,
 234                 .valid_start_item = 1,
 235                 .prev_items = (const enum rte_flow_item_type[]) {
 236                                RTE_FLOW_ITEM_TYPE_ETH,
 237                                RTE_FLOW_ITEM_TYPE_END,
 238                 },
 239         },
 240         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 241                 .copy_item = enic_fm_copy_item_ipv4,
 242                 .valid_start_item = 1,
 243                 .prev_items = (const enum rte_flow_item_type[]) {
 244                                RTE_FLOW_ITEM_TYPE_ETH,
 245                                RTE_FLOW_ITEM_TYPE_VLAN,
 246                                RTE_FLOW_ITEM_TYPE_END,
 247                 },
 248         },
 249         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 250                 .copy_item = enic_fm_copy_item_ipv6,
 251                 .valid_start_item = 1,
 252                 .prev_items = (const enum rte_flow_item_type[]) {
 253                                RTE_FLOW_ITEM_TYPE_ETH,
 254                                RTE_FLOW_ITEM_TYPE_VLAN,
 255                                RTE_FLOW_ITEM_TYPE_END,
 256                 },
 257         },
 258         [RTE_FLOW_ITEM_TYPE_UDP] = {
 259                 .copy_item = enic_fm_copy_item_udp,
 260                 .valid_start_item = 1,
 261                 .prev_items = (const enum rte_flow_item_type[]) {
 262                                RTE_FLOW_ITEM_TYPE_IPV4,
 263                                RTE_FLOW_ITEM_TYPE_IPV6,
 264                                RTE_FLOW_ITEM_TYPE_END,
 265                 },
 266         },
 267         [RTE_FLOW_ITEM_TYPE_TCP] = {
 268                 .copy_item = enic_fm_copy_item_tcp,
 269                 .valid_start_item = 1,
 270                 .prev_items = (const enum rte_flow_item_type[]) {
 271                                RTE_FLOW_ITEM_TYPE_IPV4,
 272                                RTE_FLOW_ITEM_TYPE_IPV6,
 273                                RTE_FLOW_ITEM_TYPE_END,
 274                 },
 275         },
 276         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 277                 .copy_item = enic_fm_copy_item_sctp,
 278                 .valid_start_item = 0,
 279                 .prev_items = (const enum rte_flow_item_type[]) {
 280                                RTE_FLOW_ITEM_TYPE_IPV4,
 281                                RTE_FLOW_ITEM_TYPE_IPV6,
 282                                RTE_FLOW_ITEM_TYPE_END,
 283                 },
 284         },
 285         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 286                 .copy_item = enic_fm_copy_item_vxlan,
 287                 .valid_start_item = 1,
 288                 .prev_items = (const enum rte_flow_item_type[]) {
 289                                RTE_FLOW_ITEM_TYPE_UDP,
 290                                RTE_FLOW_ITEM_TYPE_END,
 291                 },
 292         },
 293 };
 294
 295 static int
 296 enic_fm_copy_item_eth(struct copy_item_args *arg)
 297 {
 298         const struct rte_flow_item *item = arg->item;
 299         const struct rte_flow_item_eth *spec = item->spec;
 300         const struct rte_flow_item_eth *mask = item->mask;
 301         const uint8_t lvl = arg->header_level;
 302         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 303         struct fm_header_set *fm_data, *fm_mask;
 304
 305         ENICPMD_FUNC_TRACE();
 306         /* Match all if no spec */
 307         if (!spec)
 308                 return 0;
 309         if (!mask)
 310                 mask = &rte_flow_item_eth_mask;
 311         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 312         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 313         fm_data->fk_header_select |= FKH_ETHER;
 314         fm_mask->fk_header_select |= FKH_ETHER;
 315         memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
 316         memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
 317         return 0;
 318 }
 319
 320 static int
 321 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 322 {
 323         const struct rte_flow_item *item = arg->item;
 324         const struct rte_flow_item_vlan *spec = item->spec;
 325         const struct rte_flow_item_vlan *mask = item->mask;
 326         const uint8_t lvl = arg->header_level;
 327         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 328         struct fm_header_set *fm_data, *fm_mask;
 329         struct rte_ether_hdr *eth_mask;
 330         struct rte_ether_hdr *eth_val;
 331         uint32_t meta;
 332
 333         ENICPMD_FUNC_TRACE();
 334         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 335         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 336         /* Outer and inner packet vlans need different flags */
 337         meta = FKM_VLAN_PRES;
 338         if (lvl > 0)
 339                 meta = FKM_QTAG;
 340         fm_data->fk_metadata |= meta;
 341         fm_mask->fk_metadata |= meta;
 342
 343         /* Match all if no spec */
 344         if (!spec)
 345                 return 0;
 346         if (!mask)
 347                 mask = &rte_flow_item_vlan_mask;
 348
 349         eth_mask = (void *)&fm_mask->l2.eth;
 350         eth_val = (void *)&fm_data->l2.eth;
 351
 352         /* Outer TPID cannot be matched */
 353         if (eth_mask->ether_type)
 354                 return -ENOTSUP;
 355
 356         /*
 357          * When packet matching, the VIC always compares vlan-stripped
 358          * L2, regardless of vlan stripping settings. So, the inner type
 359          * from vlan becomes the ether type of the eth header.
 360          */
 361         eth_mask->ether_type = mask->inner_type;
 362         eth_val->ether_type = spec->inner_type;
 363         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 364         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 365         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 366         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 367         return 0;
 368 }
 369
 370 static int
 371 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 372 {
 373         const struct rte_flow_item *item = arg->item;
 374         const struct rte_flow_item_ipv4 *spec = item->spec;
 375         const struct rte_flow_item_ipv4 *mask = item->mask;
 376         const uint8_t lvl = arg->header_level;
 377         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 378         struct fm_header_set *fm_data, *fm_mask;
 379
 380         ENICPMD_FUNC_TRACE();
 381         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 382         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 383         fm_data->fk_metadata |= FKM_IPV4;
 384         fm_mask->fk_metadata |= FKM_IPV4;
 385
 386         if (!spec)
 387                 return 0;
 388         if (!mask)
 389                 mask = &rte_flow_item_ipv4_mask;
 390
 391         fm_data->fk_header_select |= FKH_IPV4;
 392         fm_mask->fk_header_select |= FKH_IPV4;
 393         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 394         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 395         return 0;
 396 }
 397
 398 static int
 399 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 400 {
 401         const struct rte_flow_item *item = arg->item;
 402         const struct rte_flow_item_ipv6 *spec = item->spec;
 403         const struct rte_flow_item_ipv6 *mask = item->mask;
 404         const uint8_t lvl = arg->header_level;
 405         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 406         struct fm_header_set *fm_data, *fm_mask;
 407
 408         ENICPMD_FUNC_TRACE();
 409         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 410         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 411         fm_data->fk_metadata |= FKM_IPV6;
 412         fm_mask->fk_metadata |= FKM_IPV6;
 413
 414         if (!spec)
 415                 return 0;
 416         if (!mask)
 417                 mask = &rte_flow_item_ipv6_mask;
 418
 419         fm_data->fk_header_select |= FKH_IPV6;
 420         fm_mask->fk_header_select |= FKH_IPV6;
 421         memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
 422         memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
 423         return 0;
 424 }
 425
 426 static int
 427 enic_fm_copy_item_udp(struct copy_item_args *arg)
 428 {
 429         const struct rte_flow_item *item = arg->item;
 430         const struct rte_flow_item_udp *spec = item->spec;
 431         const struct rte_flow_item_udp *mask = item->mask;
 432         const uint8_t lvl = arg->header_level;
 433         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 434         struct fm_header_set *fm_data, *fm_mask;
 435
 436         ENICPMD_FUNC_TRACE();
 437         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 438         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 439         fm_data->fk_metadata |= FKM_UDP;
 440         fm_mask->fk_metadata |= FKM_UDP;
 441
 442         if (!spec)
 443                 return 0;
 444         if (!mask)
 445                 mask = &rte_flow_item_udp_mask;
 446
 447         fm_data->fk_header_select |= FKH_UDP;
 448         fm_mask->fk_header_select |= FKH_UDP;
 449         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 450         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 451         return 0;
 452 }
 453
 454 static int
 455 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 456 {
 457         const struct rte_flow_item *item = arg->item;
 458         const struct rte_flow_item_tcp *spec = item->spec;
 459         const struct rte_flow_item_tcp *mask = item->mask;
 460         const uint8_t lvl = arg->header_level;
 461         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 462         struct fm_header_set *fm_data, *fm_mask;
 463
 464         ENICPMD_FUNC_TRACE();
 465         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 466         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 467         fm_data->fk_metadata |= FKM_TCP;
 468         fm_mask->fk_metadata |= FKM_TCP;
 469
 470         if (!spec)
 471                 return 0;
 472         if (!mask)
 473                 mask = &rte_flow_item_tcp_mask;
 474
 475         fm_data->fk_header_select |= FKH_TCP;
 476         fm_mask->fk_header_select |= FKH_TCP;
 477         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 478         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 479         return 0;
 480 }
 481
 482 static int
 483 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 484 {
 485         const struct rte_flow_item *item = arg->item;
 486         const struct rte_flow_item_sctp *spec = item->spec;
 487         const struct rte_flow_item_sctp *mask = item->mask;
 488         const uint8_t lvl = arg->header_level;
 489         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 490         struct fm_header_set *fm_data, *fm_mask;
 491         uint8_t *ip_proto_mask = NULL;
 492         uint8_t *ip_proto = NULL;
 493         uint32_t l3_fkh;
 494
 495         ENICPMD_FUNC_TRACE();
 496         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 497         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 498         /*
 499          * The NIC filter API has no flags for "match sctp", so explicitly
 500          * set the protocol number in the IP pattern.
 501          */
 502         if (fm_data->fk_metadata & FKM_IPV4) {
 503                 struct rte_ipv4_hdr *ip;
 504                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 505                 ip_proto_mask = &ip->next_proto_id;
 506                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 507                 ip_proto = &ip->next_proto_id;
 508                 l3_fkh = FKH_IPV4;
 509         } else if (fm_data->fk_metadata & FKM_IPV6) {
 510                 struct rte_ipv6_hdr *ip;
 511                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 512                 ip_proto_mask = &ip->proto;
 513                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 514                 ip_proto = &ip->proto;
 515                 l3_fkh = FKH_IPV6;
 516         } else {
 517                 /* Need IPv4/IPv6 pattern first */
 518                 return -EINVAL;
 519         }
 520         *ip_proto = IPPROTO_SCTP;
 521         *ip_proto_mask = 0xff;
 522         fm_data->fk_header_select |= l3_fkh;
 523         fm_mask->fk_header_select |= l3_fkh;
 524
 525         if (!spec)
 526                 return 0;
 527         if (!mask)
 528                 mask = &rte_flow_item_sctp_mask;
 529
 530         fm_data->fk_header_select |= FKH_L4RAW;
 531         fm_mask->fk_header_select |= FKH_L4RAW;
 532         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 533         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 534         return 0;
 535 }
 536
 537 static int
 538 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 539 {
 540         const struct rte_flow_item *item = arg->item;
 541         const struct rte_flow_item_vxlan *spec = item->spec;
 542         const struct rte_flow_item_vxlan *mask = item->mask;
 543         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 544         struct fm_header_set *fm_data, *fm_mask;
 545
 546         ENICPMD_FUNC_TRACE();
 547         /* Only 2 header levels (outer and inner) allowed */
 548         if (arg->header_level > 0)
 549                 return -EINVAL;
 550
 551         fm_data = &entry->ftm_data.fk_hdrset[0];
 552         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 553         fm_data->fk_metadata |= FKM_VXLAN;
 554         fm_mask->fk_metadata |= FKM_VXLAN;
 555         /* items from here on out are inner header items */
 556         arg->header_level = 1;
 557
 558         /* Match all if no spec */
 559         if (!spec)
 560                 return 0;
 561         if (!mask)
 562                 mask = &rte_flow_item_vxlan_mask;
 563
 564         fm_data->fk_header_select |= FKH_VXLAN;
 565         fm_mask->fk_header_select |= FKH_VXLAN;
 566         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 567         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 568         return 0;
 569 }
 570
 571 /*
 572  * Currently, raw pattern match is very limited. It is intended for matching
 573  * UDP tunnel header (e.g. vxlan or geneve).
 574  */
 575 static int
 576 enic_fm_copy_item_raw(struct copy_item_args *arg)
 577 {
 578         const struct rte_flow_item *item = arg->item;
 579         const struct rte_flow_item_raw *spec = item->spec;
 580         const struct rte_flow_item_raw *mask = item->mask;
 581         const uint8_t lvl = arg->header_level;
 582         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 583         struct fm_header_set *fm_data, *fm_mask;
 584
 585         ENICPMD_FUNC_TRACE();
 586         /* Cannot be used for inner packet */
 587         if (lvl > 0)
 588                 return -EINVAL;
 589         /* Need both spec and mask */
 590         if (!spec || !mask)
 591                 return -EINVAL;
 592         /* Only supports relative with offset 0 */
 593         if (!spec->relative || spec->offset != 0 || spec->search ||
 594             spec->limit)
 595                 return -EINVAL;
 596         /* Need non-null pattern that fits within the NIC's filter pattern */
 597         if (spec->length == 0 ||
 598             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 599             !spec->pattern || !mask->pattern)
 600                 return -EINVAL;
 601         /*
 602          * Mask fields, including length, are often set to zero. Assume that
 603          * means "same as spec" to avoid breaking existing apps. If length
 604          * is not zero, then it should be >= spec length.
 605          *
 606          * No more pattern follows this, so append to the L4 layer instead of
 607          * L5 to work with both recent and older VICs.
 608          */
 609         if (mask->length != 0 && mask->length < spec->length)
 610                 return -EINVAL;
 611
 612         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 613         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 614         fm_data->fk_header_select |= FKH_L4RAW;
 615         fm_mask->fk_header_select |= FKH_L4RAW;
 616         fm_data->fk_header_select &= ~FKH_UDP;
 617         fm_mask->fk_header_select &= ~FKH_UDP;
 618         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 619                spec->pattern, spec->length);
 620         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 621                mask->pattern, spec->length);
 622         return 0;
 623 }
 624
 625 static int
 626 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 627                struct fm_key_template *key, int entries,
 628                struct enic_fm_fet **fet_out)
 629 {
 630         struct fm_exact_match_table *cmd;
 631         struct fm_header_set *hdr;
 632         struct enic_fm_fet *fet;
 633         u64 args[3];
 634         int ret;
 635
 636         ENICPMD_FUNC_TRACE();
 637         fet = calloc(1, sizeof(struct enic_fm_fet));
 638         if (fet == NULL)
 639                 return -ENOMEM;
 640         cmd = &fm->cmd.va->fm_exact_match_table;
 641         memset(cmd, 0, sizeof(*cmd));
 642         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 643         cmd->fet_stage = FM_STAGE_LAST;
 644         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 645         if (key == NULL) {
 646                 hdr = &cmd->fet_key.fk_hdrset[0];
 647                 memset(hdr, 0, sizeof(*hdr));
 648                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 649                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 650                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 651                 hdr->l4.udp.fk_source = 0xFFFF;
 652                 hdr->l4.udp.fk_dest = 0xFFFF;
 653                 fet->default_key = 1;
 654         } else {
 655                 memcpy(&cmd->fet_key, key, sizeof(*key));
 656                 memcpy(&fet->key, key, sizeof(*key));
 657                 fet->default_key = 0;
 658         }
 659         cmd->fet_key.fk_packet_tag = 1;
 660
 661         args[0] = FM_EXACT_TABLE_ALLOC;
 662         args[1] = fm->cmd.pa;
 663         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
 664         if (ret) {
 665                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 666                 free(fet);
 667                 return ret;
 668         }
 669         fet->handle = args[0];
 670         fet->ingress = ingress;
 671         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 672                     fet->handle);
 673         *fet_out = fet;
 674         return 0;
 675 }
 676
 677 static void
 678 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 679 {
 680         ENICPMD_FUNC_TRACE();
 681         enic_fm_tbl_free(fm, fet->handle);
 682         if (!fet->default_key)
 683                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 684         free(fet);
 685 }
 686
 687 /*
 688  * Get the exact match table for the given combination of
 689  * <group, ingress, key>. Allocate one on the fly as necessary.
 690  */
 691 static int
 692 enic_fet_get(struct enic_flowman *fm,
 693              uint32_t group,
 694              uint8_t ingress,
 695              struct fm_key_template *key,
 696              struct enic_fm_fet **fet_out,
 697              struct rte_flow_error *error)
 698 {
 699         struct enic_fm_fet *fet;
 700
 701         ENICPMD_FUNC_TRACE();
 702         /* See if we already have this table open */
 703         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 704                 if (fet->group == group && fet->ingress == ingress)
 705                         break;
 706         }
 707         if (fet == NULL) {
 708                 /* Jumping to a non-existing group? Use the default table */
 709                 if (key == NULL) {
 710                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 711                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 712                         return rte_flow_error_set(error, EINVAL,
 713                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 714                                 NULL, "enic: cannot get exact match table");
 715                 }
 716                 fet->group = group;
 717                 /* Default table is never on the open table list */
 718                 if (!fet->default_key)
 719                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 720         }
 721         fet->ref++;
 722         *fet_out = fet;
 723         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 724                     fet->default_key ? "default" : "",
 725                     fet->ingress ? "ingress" : "egress",
 726                     fet->group, fet->ref);
 727         return 0;
 728 }
 729
 730 static void
 731 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 732 {
 733         ENICPMD_FUNC_TRACE();
 734         RTE_ASSERT(fet->ref > 0);
 735         fet->ref--;
 736         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 737                     fet->default_key ? "default" : "",
 738                     fet->ingress ? "ingress" : "egress",
 739                     fet->group, fet->ref);
 740         if (fet->ref == 0)
 741                 enic_fet_free(fm, fet);
 742 }
 743
 744 /* Return 1 if current item is valid on top of the previous one. */
 745 static int
 746 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 747                        const struct enic_fm_items *item_info,
 748                        uint8_t is_first_item)
 749 {
 750         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 751
 752         ENICPMD_FUNC_TRACE();
 753         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 754                 if (prev_item == *allowed_items)
 755                         return 1;
 756         }
 757
 758         /* This is the first item in the stack. Check if that's cool */
 759         if (is_first_item && item_info->valid_start_item)
 760                 return 1;
 761         return 0;
 762 }
 763
 764 /*
 765  * Build the flow manager match entry structure from the provided pattern.
 766  * The pattern is validated as the items are copied.
 767  */
 768 static int
 769 enic_fm_copy_entry(struct enic_flowman *fm,
 770                    const struct rte_flow_item pattern[],
 771                    struct rte_flow_error *error)
 772 {
 773         const struct enic_fm_items *item_info;
 774         enum rte_flow_item_type prev_item;
 775         const struct rte_flow_item *item;
 776         struct copy_item_args args;
 777         uint8_t prev_header_level;
 778         uint8_t is_first_item;
 779         int ret;
 780
 781         ENICPMD_FUNC_TRACE();
 782         item = pattern;
 783         is_first_item = 1;
 784         prev_item = RTE_FLOW_ITEM_TYPE_END;
 785
 786         args.fm_tcam_entry = &fm->tcam_entry;
 787         args.header_level = 0;
 788         prev_header_level = 0;
 789         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 790                 /*
 791                  * Get info about how to validate and copy the item. If NULL
 792                  * is returned the nic does not support the item.
 793                  */
 794                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 795                         continue;
 796
 797                 item_info = &enic_fm_items[item->type];
 798
 799                 if (item->type > FM_MAX_ITEM_TYPE ||
 800                     item_info->copy_item == NULL) {
 801                         return rte_flow_error_set(error, ENOTSUP,
 802                                 RTE_FLOW_ERROR_TYPE_ITEM,
 803                                 NULL, "enic: unsupported item");
 804                 }
 805
 806                 /* check to see if item stacking is valid */
 807                 if (!fm_item_stacking_valid(prev_item, item_info,
 808                                             is_first_item))
 809                         goto stacking_error;
 810
 811                 args.item = item;
 812                 ret = item_info->copy_item(&args);
 813                 if (ret)
 814                         goto item_not_supported;
 815                 /* Going from outer to inner? Treat it as a new packet start */
 816                 if (prev_header_level != args.header_level) {
 817                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 818                         is_first_item = 1;
 819                 } else {
 820                         prev_item = item->type;
 821                         is_first_item = 0;
 822                 }
 823                 prev_header_level = args.header_level;
 824         }
 825         return 0;
 826
 827 item_not_supported:
 828         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 829                                   NULL, "enic: unsupported item type");
 830
 831 stacking_error:
 832         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 833                                   item, "enic: unsupported item stack");
 834 }
 835
 836 static void
 837 flow_item_skip_void(const struct rte_flow_item **item)
 838 {
 839         for ( ; ; (*item)++)
 840                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 841                         return;
 842 }
 843
 844 static void
 845 append_template(void **template, uint8_t *off, const void *data, int len)
 846 {
 847         memcpy(*template, data, len);
 848         *template = (char *)*template + len;
 849         *off = *off + len;
 850 }
 851
 852 static int
 853 enic_fm_append_action_op(struct enic_flowman *fm,
 854                          struct fm_action_op *fm_op,
 855                          struct rte_flow_error *error)
 856 {
 857         int count;
 858
 859         count = fm->action_op_count;
 860         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 861                     count, fm_op->fa_op);
 862         if (count == FM_ACTION_OP_MAX) {
 863                 return rte_flow_error_set(error, EINVAL,
 864                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 865                         "too many action operations");
 866         }
 867         fm->action.fma_action_ops[count] = *fm_op;
 868         fm->action_op_count = count + 1;
 869         return 0;
 870 }
 871
 872 /* Steer operations need to appear before other ops */
 873 static void
 874 enic_fm_reorder_action_op(struct enic_flowman *fm)
 875 {
 876         struct fm_action_op *dst, *dst_head, *src, *src_head;
 877
 878         ENICPMD_FUNC_TRACE();
 879         /* Move steer ops to the front. */
 880         src = fm->action.fma_action_ops;
 881         src_head = src;
 882         dst = fm->action_tmp.fma_action_ops;
 883         dst_head = dst;
 884         /* Copy steer ops to tmp */
 885         while (src->fa_op != FMOP_END) {
 886                 if (src->fa_op == FMOP_RQ_STEER) {
 887                         ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
 888                                     (long)(src - src_head),
 889                                     (long)(dst - dst_head));
 890                         *dst = *src;
 891                         dst++;
 892                 }
 893                 src++;
 894         }
 895         /* Then append non-steer ops */
 896         src = src_head;
 897         while (src->fa_op != FMOP_END) {
 898                 if (src->fa_op != FMOP_RQ_STEER) {
 899                         ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
 900                                     (long)(src - src_head),
 901                                     (long)(dst - dst_head));
 902                         *dst = *src;
 903                         dst++;
 904                 }
 905                 src++;
 906         }
 907         /* Copy END */
 908         *dst = *src;
 909         /* Finally replace the original action with the reordered one */
 910         memcpy(fm->action.fma_action_ops, fm->action_tmp.fma_action_ops,
 911                sizeof(fm->action.fma_action_ops));
 912 }
 913
 914 /* VXLAN decap is done via flowman compound action */
 915 static int
 916 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 917                          struct fm_tcam_match_entry *fmt,
 918                          const struct rte_flow_action *action,
 919                          struct rte_flow_error *error)
 920 {
 921         struct fm_header_set *fm_data;
 922         struct fm_action_op fm_op;
 923
 924         ENICPMD_FUNC_TRACE();
 925         fm_data = &fmt->ftm_data.fk_hdrset[0];
 926         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 927                 return rte_flow_error_set(error, EINVAL,
 928                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 929                         "vxlan-decap: vxlan must be in pattern");
 930         }
 931
 932         memset(&fm_op, 0, sizeof(fm_op));
 933         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
 934         return enic_fm_append_action_op(fm, &fm_op, error);
 935 }
 936
 937 /* VXLAN encap is done via flowman compound action */
 938 static int
 939 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
 940                          const struct rte_flow_item *item,
 941                          struct rte_flow_error *error)
 942 {
 943         struct fm_action_op fm_op;
 944         struct rte_ether_hdr *eth;
 945         uint16_t *ethertype;
 946         void *template;
 947         uint8_t off;
 948
 949         ENICPMD_FUNC_TRACE();
 950         memset(&fm_op, 0, sizeof(fm_op));
 951         fm_op.fa_op = FMOP_ENCAP;
 952         template = fm->action.fma_data;
 953         off = 0;
 954         /*
 955          * Copy flow items to the flowman template starting L2.
 956          * L2 must be ethernet.
 957          */
 958         flow_item_skip_void(&item);
 959         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
 960                 return rte_flow_error_set(error, EINVAL,
 961                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 962                         "vxlan-encap: first item should be ethernet");
 963         eth = (struct rte_ether_hdr *)template;
 964         ethertype = &eth->ether_type;
 965         append_template(&template, &off, item->spec,
 966                         sizeof(struct rte_flow_item_eth));
 967         item++;
 968         flow_item_skip_void(&item);
 969         /* Optional VLAN */
 970         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
 971                 const struct rte_flow_item_vlan *spec;
 972
 973                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
 974                 spec = item->spec;
 975                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
 976                 item++;
 977                 flow_item_skip_void(&item);
 978         }
 979         /* L3 must be IPv4, IPv6 */
 980         switch (item->type) {
 981         case RTE_FLOW_ITEM_TYPE_IPV4:
 982         {
 983                 struct rte_ipv4_hdr *ip4;
 984
 985                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
 986                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 987                 ip4 = (struct rte_ipv4_hdr *)template;
 988                 /*
 989                  * Offset of IPv4 length field and its initial value
 990                  * (IP + UDP + VXLAN) are specified in the action. The NIC
 991                  * will add inner packet length.
 992                  */
 993                 fm_op.encap.len1_offset = off +
 994                         offsetof(struct rte_ipv4_hdr, total_length);
 995                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
 996                         sizeof(struct rte_udp_hdr) +
 997                         sizeof(struct rte_vxlan_hdr);
 998                 append_template(&template, &off, item->spec,
 999                                 sizeof(struct rte_ipv4_hdr));
1000                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1001                 if (ip4->time_to_live == 0)
1002                         ip4->time_to_live = IP_DEFTTL;
1003                 ip4->next_proto_id = IPPROTO_UDP;
1004                 break;
1005         }
1006         case RTE_FLOW_ITEM_TYPE_IPV6:
1007         {
1008                 struct rte_ipv6_hdr *ip6;
1009
1010                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1011                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1012                 ip6 = (struct rte_ipv6_hdr *)template;
1013                 fm_op.encap.len1_offset = off +
1014                         offsetof(struct rte_ipv6_hdr, payload_len);
1015                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1016                         sizeof(struct rte_vxlan_hdr);
1017                 append_template(&template, &off, item->spec,
1018                                 sizeof(struct rte_ipv6_hdr));
1019                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1020                 if (ip6->hop_limits == 0)
1021                         ip6->hop_limits = IP_DEFTTL;
1022                 ip6->proto = IPPROTO_UDP;
1023                 break;
1024         }
1025         default:
1026                 return rte_flow_error_set(error,
1027                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1028                         "vxlan-encap: L3 must be IPv4/IPv6");
1029         }
1030         item++;
1031         flow_item_skip_void(&item);
1032
1033         /* L4 is UDP */
1034         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1035                 return rte_flow_error_set(error, EINVAL,
1036                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1037                         "vxlan-encap: UDP must follow IPv4/IPv6");
1038         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1039         fm_op.encap.len2_offset =
1040                 off + offsetof(struct rte_udp_hdr, dgram_len);
1041         fm_op.encap.len2_delta =
1042                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1043         append_template(&template, &off, item->spec,
1044                         sizeof(struct rte_udp_hdr));
1045         item++;
1046         flow_item_skip_void(&item);
1047
1048         /* Finally VXLAN */
1049         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1050                 return rte_flow_error_set(error,
1051                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1052                         "vxlan-encap: VXLAN must follow UDP");
1053         append_template(&template, &off, item->spec,
1054                         sizeof(struct rte_flow_item_vxlan));
1055
1056         /*
1057          * Fill in the rest of the action structure.
1058          * Indicate that we want to encap with vxlan at packet start.
1059          */
1060         fm_op.encap.template_offset = 0;
1061         fm_op.encap.template_len = off;
1062         return enic_fm_append_action_op(fm, &fm_op, error);
1063 }
1064
1065 static int
1066 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1067                   uint64_t *handle)
1068 {
1069         uint32_t bdf;
1070         u64 args[2];
1071         int rc;
1072
1073         ENICPMD_FUNC_TRACE();
1074         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1075                     addr->function);
1076         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1077         args[0] = FM_VNIC_FIND;
1078         args[1] = bdf;
1079         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1080         if (rc != 0) {
1081                 ENICPMD_LOG(ERR, "allocating counters rc=%d", rc);
1082                 return rc;
1083         }
1084         *handle = args[0];
1085         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1086         return 0;
1087 }
1088
1089 /* Translate flow actions to flowman TCAM entry actions */
1090 static int
1091 enic_fm_copy_action(struct enic_flowman *fm,
1092                     const struct rte_flow_action actions[],
1093                     uint8_t ingress,
1094                     struct rte_flow_error *error)
1095 {
1096         enum {
1097                 FATE = 1 << 0,
1098                 MARK = 1 << 1,
1099                 PASSTHRU = 1 << 2,
1100                 COUNT = 1 << 3,
1101                 ENCAP = 1 << 4,
1102         };
1103         struct fm_tcam_match_entry *fmt;
1104         struct fm_action_op fm_op;
1105         struct enic *enic;
1106         uint32_t overlap;
1107         uint64_t vnic_h;
1108         bool first_rq;
1109         int ret;
1110
1111         ENICPMD_FUNC_TRACE();
1112         fmt = &fm->tcam_entry;
1113         first_rq = true;
1114         enic = fm->enic;
1115         overlap = 0;
1116         vnic_h = 0; /* 0 = current vNIC */
1117         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1118                 switch (actions->type) {
1119                 case RTE_FLOW_ACTION_TYPE_VOID:
1120                         continue;
1121                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1122                         if (overlap & PASSTHRU)
1123                                 goto unsupported;
1124                         overlap |= PASSTHRU;
1125                         break;
1126                 }
1127                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1128                         const struct rte_flow_action_jump *jump =
1129                                 actions->conf;
1130                         struct enic_fm_fet *fet;
1131
1132                         if (overlap & FATE)
1133                                 goto unsupported;
1134                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1135                                            &fet, error);
1136                         if (ret)
1137                                 return ret;
1138                         overlap |= FATE;
1139                         memset(&fm_op, 0, sizeof(fm_op));
1140                         fm_op.fa_op = FMOP_EXACT_MATCH;
1141                         fm_op.exact.handle = fet->handle;
1142                         fm->fet = fet;
1143                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1144                         if (ret)
1145                                 return ret;
1146                         break;
1147                 }
1148                 case RTE_FLOW_ACTION_TYPE_MARK: {
1149                         const struct rte_flow_action_mark *mark =
1150                                 actions->conf;
1151
1152                         if (overlap & MARK)
1153                                 goto unsupported;
1154                         overlap |= MARK;
1155                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1156                                 return rte_flow_error_set(error, EINVAL,
1157                                         RTE_FLOW_ERROR_TYPE_ACTION,
1158                                         NULL, "invalid mark id");
1159                         memset(&fm_op, 0, sizeof(fm_op));
1160                         fm_op.fa_op = FMOP_MARK;
1161                         fm_op.mark.mark = mark->id + 1;
1162                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1163                         if (ret)
1164                                 return ret;
1165                         break;
1166                 }
1167                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1168                         if (overlap & MARK)
1169                                 goto unsupported;
1170                         overlap |= MARK;
1171                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1172                         memset(&fm_op, 0, sizeof(fm_op));
1173                         fm_op.fa_op = FMOP_MARK;
1174                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1175                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1176                         if (ret)
1177                                 return ret;
1178                         break;
1179                 }
1180                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1181                         const struct rte_flow_action_queue *queue =
1182                                 actions->conf;
1183
1184                         /*
1185                          * If other fate kind is set, fail.  Multiple
1186                          * queue actions are ok.
1187                          */
1188                         if ((overlap & FATE) && first_rq)
1189                                 goto unsupported;
1190                         first_rq = false;
1191                         overlap |= FATE;
1192                         memset(&fm_op, 0, sizeof(fm_op));
1193                         fm_op.fa_op = FMOP_RQ_STEER;
1194                         fm_op.rq_steer.rq_index =
1195                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1196                         fm_op.rq_steer.vnic_handle = vnic_h;
1197                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1198                         if (ret)
1199                                 return ret;
1200                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1201                                     fm_op.rq_steer.rq_index);
1202                         break;
1203                 }
1204                 case RTE_FLOW_ACTION_TYPE_DROP: {
1205                         if (overlap & FATE)
1206                                 goto unsupported;
1207                         overlap |= FATE;
1208                         memset(&fm_op, 0, sizeof(fm_op));
1209                         fm_op.fa_op = FMOP_DROP;
1210                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1211                         if (ret)
1212                                 return ret;
1213                         ENICPMD_LOG(DEBUG, "create DROP action");
1214                         break;
1215                 }
1216                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1217                         if (overlap & COUNT)
1218                                 goto unsupported;
1219                         overlap |= COUNT;
1220                         /* Count is associated with entry not action on VIC. */
1221                         fmt->ftm_flags |= FMEF_COUNTER;
1222                         break;
1223                 }
1224                 case RTE_FLOW_ACTION_TYPE_RSS: {
1225                         const struct rte_flow_action_rss *rss = actions->conf;
1226                         bool allow;
1227                         uint16_t i;
1228
1229                         /*
1230                          * Hardware does not support general RSS actions, but
1231                          * we can still support the dummy one that is used to
1232                          * "receive normally".
1233                          */
1234                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1235                                 rss->level == 0 &&
1236                                 (rss->types == 0 ||
1237                                  rss->types == enic->rss_hf) &&
1238                                 rss->queue_num == enic->rq_count &&
1239                                 rss->key_len == 0;
1240                         /* Identity queue map is ok */
1241                         for (i = 0; i < rss->queue_num; i++)
1242                                 allow = allow && (i == rss->queue[i]);
1243                         if (!allow)
1244                                 goto unsupported;
1245                         if (overlap & FATE)
1246                                 goto unsupported;
1247                         /* Need MARK or FLAG */
1248                         if (!(overlap & MARK))
1249                                 goto unsupported;
1250                         overlap |= FATE;
1251                         break;
1252                 }
1253                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1254                         const struct rte_flow_action_port_id *port;
1255                         struct rte_pci_device *pdev;
1256                         struct rte_eth_dev *dev;
1257
1258                         port = actions->conf;
1259                         if (port->original) {
1260                                 vnic_h = 0; /* This port */
1261                                 break;
1262                         }
1263                         ENICPMD_LOG(DEBUG, "port id %u", port->id);
1264                         if (!rte_eth_dev_is_valid_port(port->id)) {
1265                                 return rte_flow_error_set(error, EINVAL,
1266                                         RTE_FLOW_ERROR_TYPE_ACTION,
1267                                         NULL, "invalid port_id");
1268                         }
1269                         dev = &rte_eth_devices[port->id];
1270                         if (!dev_is_enic(dev)) {
1271                                 return rte_flow_error_set(error, EINVAL,
1272                                         RTE_FLOW_ERROR_TYPE_ACTION,
1273                                         NULL, "port_id is not enic");
1274                         }
1275                         pdev = RTE_ETH_DEV_TO_PCI(dev);
1276                         if (enic_fm_find_vnic(enic, &pdev->addr, &vnic_h)) {
1277                                 return rte_flow_error_set(error, EINVAL,
1278                                         RTE_FLOW_ERROR_TYPE_ACTION,
1279                                         NULL, "port_id is not vnic");
1280                         }
1281                         break;
1282                 }
1283                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1284                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1285                                 error);
1286                         if (ret != 0)
1287                                 return ret;
1288                         break;
1289                 }
1290                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1291                         const struct rte_flow_action_vxlan_encap *encap;
1292
1293                         encap = actions->conf;
1294                         if (overlap & ENCAP)
1295                                 goto unsupported;
1296                         overlap |= ENCAP;
1297                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1298                                 error);
1299                         if (ret != 0)
1300                                 return ret;
1301                         break;
1302                 }
1303                 default:
1304                         goto unsupported;
1305                 }
1306         }
1307
1308         if (!(overlap & (FATE | PASSTHRU | COUNT)))
1309                 goto unsupported;
1310         memset(&fm_op, 0, sizeof(fm_op));
1311         fm_op.fa_op = FMOP_END;
1312         ret = enic_fm_append_action_op(fm, &fm_op, error);
1313         if (ret)
1314                 return ret;
1315         enic_fm_reorder_action_op(fm);
1316         return 0;
1317
1318 unsupported:
1319         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1320                                   NULL, "enic: unsupported action");
1321 }
1322
1323 /** Check if the action is supported */
1324 static int
1325 enic_fm_match_action(const struct rte_flow_action *action,
1326                      const enum rte_flow_action_type *supported_actions)
1327 {
1328         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1329              supported_actions++) {
1330                 if (action->type == *supported_actions)
1331                         return 1;
1332         }
1333         return 0;
1334 }
1335
1336 /* Debug function to dump internal NIC action structure. */
1337 static void
1338 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1339 {
1340         /* Manually keep in sync with FMOP commands */
1341         const char *fmop_str[FMOP_OP_MAX] = {
1342                 [FMOP_END] = "end",
1343                 [FMOP_DROP] = "drop",
1344                 [FMOP_RQ_STEER] = "steer",
1345                 [FMOP_EXACT_MATCH] = "exmatch",
1346                 [FMOP_MARK] = "mark",
1347                 [FMOP_EXT_MARK] = "ext_mark",
1348                 [FMOP_TAG] = "tag",
1349                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1350                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1351                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1352                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1353                 [FMOP_ENCAP] = "encap",
1354                 [FMOP_SET_OVLAN] = "set_ovlan",
1355                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1356         };
1357         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1358         char buf[128], *bp = buf;
1359         const char *op_str;
1360         int i, n, buf_len;
1361
1362         buf[0] = '\0';
1363         buf_len = sizeof(buf);
1364         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1365                 if (op->fa_op == FMOP_END)
1366                         break;
1367                 if (op->fa_op >= FMOP_OP_MAX)
1368                         op_str = "unknown";
1369                 else
1370                         op_str = fmop_str[op->fa_op];
1371                 n = snprintf(bp, buf_len, "%s,", op_str);
1372                 if (n > 0 && n < buf_len) {
1373                         bp += n;
1374                         buf_len -= n;
1375                 }
1376                 op++;
1377         }
1378         /* Remove trailing comma */
1379         if (buf[0])
1380                 *(bp - 1) = '\0';
1381         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1382 }
1383
1384 static int
1385 bits_to_str(uint32_t bits, const char *strings[], int max,
1386             char *buf, int buf_len)
1387 {
1388         int i, n = 0, len = 0;
1389
1390         for (i = 0; i < max; i++) {
1391                 if (bits & (1 << i)) {
1392                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1393                         if (n > 0 && n < buf_len) {
1394                                 buf += n;
1395                                 buf_len -= n;
1396                                 len += n;
1397                         }
1398                 }
1399         }
1400         /* Remove trailing comma */
1401         if (len) {
1402                 *(buf - 1) = '\0';
1403                 len--;
1404         }
1405         return len;
1406 }
1407
1408 /* Debug function to dump internal NIC filter structure. */
1409 static void
1410 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1411                           int buf_len)
1412 {
1413         /* Manually keep in sync with FKM_BITS */
1414         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1415                 [FKM_QTAG_BIT] = "qtag",
1416                 [FKM_CMD_BIT] = "cmd",
1417                 [FKM_IPV4_BIT] = "ip4",
1418                 [FKM_IPV6_BIT] = "ip6",
1419                 [FKM_ROCE_BIT] = "roce",
1420                 [FKM_UDP_BIT] = "udp",
1421                 [FKM_TCP_BIT] = "tcp",
1422                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1423                 [FKM_IPFRAG_BIT] = "ipfrag",
1424                 [FKM_NVGRE_BIT] = "nvgre",
1425                 [FKM_VXLAN_BIT] = "vxlan",
1426                 [FKM_GENEVE_BIT] = "geneve",
1427                 [FKM_NSH_BIT] = "nsh",
1428                 [FKM_ROCEV2_BIT] = "rocev2",
1429                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1430                 [FKM_IPOK_BIT] = "ipok",
1431                 [FKM_L4OK_BIT] = "l4ok",
1432                 [FKM_ROCEOK_BIT] = "roceok",
1433                 [FKM_FCSOK_BIT] = "fcsok",
1434                 [FKM_EG_SPAN_BIT] = "eg_span",
1435                 [FKM_IG_SPAN_BIT] = "ig_span",
1436                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1437         };
1438         /* Manually keep in sync with FKH_BITS */
1439         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1440                 [FKH_ETHER_BIT] = "eth",
1441                 [FKH_QTAG_BIT] = "qtag",
1442                 [FKH_L2RAW_BIT] = "l2raw",
1443                 [FKH_IPV4_BIT] = "ip4",
1444                 [FKH_IPV6_BIT] = "ip6",
1445                 [FKH_L3RAW_BIT] = "l3raw",
1446                 [FKH_UDP_BIT] = "udp",
1447                 [FKH_TCP_BIT] = "tcp",
1448                 [FKH_ICMP_BIT] = "icmp",
1449                 [FKH_VXLAN_BIT] = "vxlan",
1450                 [FKH_L4RAW_BIT] = "l4raw",
1451         };
1452         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1453         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1454         int n;
1455
1456         if (!fkm_bits && !fkh_bits)
1457                 return;
1458         n = snprintf(buf, buf_len, "metadata(");
1459         if (n > 0 && n < buf_len) {
1460                 buf += n;
1461                 buf_len -= n;
1462         }
1463         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1464         if (n > 0 && n < buf_len) {
1465                 buf += n;
1466                 buf_len -= n;
1467         }
1468         n = snprintf(buf, buf_len, ") valid hdr fields(");
1469         if (n > 0 && n < buf_len) {
1470                 buf += n;
1471                 buf_len -= n;
1472         }
1473         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1474         if (n > 0 && n < buf_len) {
1475                 buf += n;
1476                 buf_len -= n;
1477         }
1478         snprintf(buf, buf_len, ")");
1479 }
1480
1481 static void
1482 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1483                         uint8_t ingress)
1484 {
1485         char buf[256];
1486
1487         memset(buf, 0, sizeof(buf));
1488         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1489                                   buf, sizeof(buf));
1490         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1491                     (ingress) ? "IG" : "EG", buf,
1492                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1493         memset(buf, 0, sizeof(buf));
1494         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1495                                   buf, sizeof(buf));
1496         if (buf[0])
1497                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1498 }
1499
1500 /* Debug function to dump internal NIC flow structures. */
1501 static void
1502 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1503                         const struct fm_action *fm_action,
1504                         uint8_t ingress)
1505 {
1506         if (rte_log_get_level(enic_pmd_logtype) < (int)RTE_LOG_DEBUG)
1507                 return;
1508         enic_fm_dump_tcam_match(fm_match, ingress);
1509         enic_fm_dump_tcam_actions(fm_action);
1510 }
1511
1512 static int
1513 enic_fm_flow_parse(struct enic_flowman *fm,
1514                    const struct rte_flow_attr *attrs,
1515                    const struct rte_flow_item pattern[],
1516                    const struct rte_flow_action actions[],
1517                    struct rte_flow_error *error)
1518 {
1519         const struct rte_flow_action *action;
1520         unsigned int ret;
1521         static const enum rte_flow_action_type *sa;
1522
1523         ENICPMD_FUNC_TRACE();
1524         ret = 0;
1525         if (!pattern) {
1526                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1527                                    NULL, "no pattern specified");
1528                 return -rte_errno;
1529         }
1530
1531         if (!actions) {
1532                 rte_flow_error_set(error, EINVAL,
1533                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1534                                    NULL, "no action specified");
1535                 return -rte_errno;
1536         }
1537
1538         if (attrs) {
1539                 if (attrs->priority) {
1540                         rte_flow_error_set(error, ENOTSUP,
1541                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1542                                            NULL,
1543                                            "priorities are not supported");
1544                         return -rte_errno;
1545                 } else if (attrs->transfer) {
1546                         rte_flow_error_set(error, ENOTSUP,
1547                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1548                                            NULL,
1549                                            "transfer is not supported");
1550                         return -rte_errno;
1551                 } else if (attrs->ingress && attrs->egress) {
1552                         rte_flow_error_set(error, ENOTSUP,
1553                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1554                                            NULL,
1555                                            "bidirectional rules not supported");
1556                         return -rte_errno;
1557                 }
1558
1559         } else {
1560                 rte_flow_error_set(error, EINVAL,
1561                                    RTE_FLOW_ERROR_TYPE_ATTR,
1562                                    NULL, "no attribute specified");
1563                 return -rte_errno;
1564         }
1565
1566         /* Verify Actions. */
1567         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1568              enic_fm_supported_eg_actions;
1569         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1570              action++) {
1571                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1572                         continue;
1573                 else if (!enic_fm_match_action(action, sa))
1574                         break;
1575         }
1576         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1577                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1578                                    action, "invalid action");
1579                 return -rte_errno;
1580         }
1581         ret = enic_fm_copy_entry(fm, pattern, error);
1582         if (ret)
1583                 return ret;
1584         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1585         return ret;
1586 }
1587
1588 static void
1589 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1590 {
1591         if (!fm_flow->counter_valid)
1592                 return;
1593         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1594         fm_flow->counter_valid = false;
1595 }
1596
1597 static int
1598 enic_fm_more_counters(struct enic_flowman *fm)
1599 {
1600         struct enic_fm_counter *new_stack;
1601         struct enic_fm_counter *ctrs;
1602         struct enic *enic;
1603         int i, rc;
1604         u64 args[2];
1605
1606         ENICPMD_FUNC_TRACE();
1607         enic = fm->enic;
1608         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1609                                 FM_COUNTERS_EXPAND) *
1610                                 sizeof(struct enic_fm_counter), 0);
1611         if (new_stack == NULL) {
1612                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1613                 return -ENOMEM;
1614         }
1615         fm->counter_stack = new_stack;
1616
1617         args[0] = FM_COUNTER_BRK;
1618         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1619         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1620         if (rc != 0) {
1621                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1622                 return rc;
1623         }
1624         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1625                 fm->counters_alloced;
1626         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1627                 ctrs->handle = fm->counters_alloced + i;
1628                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1629         }
1630         fm->counters_alloced += FM_COUNTERS_EXPAND;
1631         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1632                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1633         return 0;
1634 }
1635
1636 static int
1637 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1638 {
1639         struct enic *enic;
1640         u64 args[3];
1641         int ret;
1642
1643         ENICPMD_FUNC_TRACE();
1644         enic = fm->enic;
1645         args[0] = FM_COUNTER_QUERY;
1646         args[1] = c->handle;
1647         args[2] = 1; /* clear */
1648         ret = vnic_dev_flowman_cmd(enic->vdev, args, 3);
1649         if (ret) {
1650                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1651                             ret, c->handle);
1652                 return ret;
1653         }
1654         return 0;
1655 }
1656
1657 static int
1658 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1659                       struct enic_fm_counter **ctr)
1660 {
1661         struct enic_fm_counter *c;
1662         int ret;
1663
1664         ENICPMD_FUNC_TRACE();
1665         *ctr = NULL;
1666         if (SLIST_EMPTY(&fm->counters)) {
1667                 ret = enic_fm_more_counters(fm);
1668                 if (ret)
1669                         return rte_flow_error_set(error, -ret,
1670                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1671                                 NULL, "enic: out of counters");
1672         }
1673         c = SLIST_FIRST(&fm->counters);
1674         SLIST_REMOVE_HEAD(&fm->counters, next);
1675         *ctr = c;
1676         return 0;
1677 }
1678
1679 static int
1680 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1681 {
1682         u64 args[2];
1683         int rc;
1684
1685         ENICPMD_FUNC_TRACE();
1686         args[0] = FM_ACTION_FREE;
1687         args[1] = handle;
1688         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1689         if (rc)
1690                 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1691                             rc, handle);
1692         return rc;
1693 }
1694
1695 static int
1696 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1697 {
1698         u64 args[2];
1699         int rc;
1700
1701         ENICPMD_FUNC_TRACE();
1702         args[0] = FM_MATCH_ENTRY_REMOVE;
1703         args[1] = handle;
1704         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1705         if (rc)
1706                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1707                             " handle=0x%" PRIx64, rc, handle);
1708         return rc;
1709 }
1710
1711 static struct enic_fm_jump_flow *
1712 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1713 {
1714         struct enic_fm_jump_flow *j;
1715
1716         ENICPMD_FUNC_TRACE();
1717         TAILQ_FOREACH(j, &fm->jump_list, list) {
1718                 if (j->group == group)
1719                         return j;
1720         }
1721         return NULL;
1722 }
1723
1724 static void
1725 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1726 {
1727         struct enic_fm_jump_flow *j;
1728
1729         ENICPMD_FUNC_TRACE();
1730         TAILQ_FOREACH(j, &fm->jump_list, list) {
1731                 if (j->flow == flow) {
1732                         TAILQ_REMOVE(&fm->jump_list, j, list);
1733                         free(j);
1734                         return;
1735                 }
1736         }
1737 }
1738
1739 static int
1740 save_jump_flow(struct enic_flowman *fm,
1741                struct rte_flow *flow,
1742                uint32_t group,
1743                struct fm_tcam_match_entry *match,
1744                struct fm_action *action)
1745 {
1746         struct enic_fm_jump_flow *j;
1747
1748         ENICPMD_FUNC_TRACE();
1749         j = calloc(1, sizeof(struct enic_fm_jump_flow));
1750         if (j == NULL)
1751                 return -ENOMEM;
1752         j->flow = flow;
1753         j->group = group;
1754         j->match = *match;
1755         j->action = *action;
1756         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1757         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1758         return 0;
1759 }
1760
1761 static void
1762 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1763 {
1764         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1765                 enic_fm_entry_free(fm, fm_flow->entry_handle);
1766                 fm_flow->entry_handle = FM_INVALID_HANDLE;
1767         }
1768         if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1769                 enic_fm_action_free(fm, fm_flow->action_handle);
1770                 fm_flow->action_handle = FM_INVALID_HANDLE;
1771         }
1772         enic_fm_counter_free(fm, fm_flow);
1773         if (fm_flow->fet) {
1774                 enic_fet_put(fm, fm_flow->fet);
1775                 fm_flow->fet = NULL;
1776         }
1777 }
1778
1779 static void
1780 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1781 {
1782         if (flow->fm->fet && flow->fm->fet->default_key)
1783                 remove_jump_flow(fm, flow);
1784         __enic_fm_flow_free(fm, flow->fm);
1785         free(flow->fm);
1786         free(flow);
1787 }
1788
1789 static int
1790 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1791                        struct fm_tcam_match_entry *match_in,
1792                        uint64_t *entry_handle,
1793                        uint8_t ingress,
1794                        struct rte_flow_error *error)
1795 {
1796         struct fm_tcam_match_entry *ftm;
1797         u64 args[3];
1798         int ret;
1799
1800         ENICPMD_FUNC_TRACE();
1801         /* Copy entry to the command buffer */
1802         ftm = &fm->cmd.va->fm_tcam_match_entry;
1803         memcpy(ftm, match_in, sizeof(*ftm));
1804         /* Add TCAM entry */
1805         args[0] = FM_TCAM_ENTRY_INSTALL;
1806         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1807         args[2] = fm->cmd.pa;
1808         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1809         if (ret != 0) {
1810                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1811                             ingress ? "ingress" : "egress", ret);
1812                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1813                         NULL, "enic: devcmd(tcam-entry-install)");
1814                 return ret;
1815         }
1816         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1817                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
1818         *entry_handle = args[0];
1819         return 0;
1820 }
1821
1822 static int
1823 enic_fm_add_exact_entry(struct enic_flowman *fm,
1824                         struct fm_tcam_match_entry *match_in,
1825                         uint64_t *entry_handle,
1826                         struct enic_fm_fet *fet,
1827                         struct rte_flow_error *error)
1828 {
1829         struct fm_exact_match_entry *fem;
1830         u64 args[3];
1831         int ret;
1832
1833         ENICPMD_FUNC_TRACE();
1834         /* The new entry must have the table's key */
1835         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1836                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1837                 return rte_flow_error_set(error, EINVAL,
1838                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1839                         "enic: key does not match group's key");
1840         }
1841
1842         /* Copy entry to the command buffer */
1843         fem = &fm->cmd.va->fm_exact_match_entry;
1844         /*
1845          * Translate TCAM entry to exact entry. As is only need to drop
1846          * position and mask. The mask is part of the exact match table.
1847          * Position (aka priority) is not supported in the exact match table.
1848          */
1849         fem->fem_data = match_in->ftm_data;
1850         fem->fem_flags = match_in->ftm_flags;
1851         fem->fem_action = match_in->ftm_action;
1852         fem->fem_counter = match_in->ftm_counter;
1853
1854         /* Add exact entry */
1855         args[0] = FM_EXACT_ENTRY_INSTALL;
1856         args[1] = fet->handle;
1857         args[2] = fm->cmd.pa;
1858         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1859         if (ret != 0) {
1860                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
1861                             fet->ingress ? "ingress" : "egress", fet->group);
1862                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1863                         NULL, "enic: devcmd(exact-entry-install)");
1864                 return ret;
1865         }
1866         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
1867                     " handle=0x%" PRIx64,
1868                     fet->ingress ? "ingress" : "egress", fet->group,
1869                     (uint64_t)args[0]);
1870         *entry_handle = args[0];
1871         return 0;
1872 }
1873
1874 /* Push match-action to the NIC. */
1875 static int
1876 __enic_fm_flow_add_entry(struct enic_flowman *fm,
1877                          struct enic_fm_flow *fm_flow,
1878                          struct fm_tcam_match_entry *match_in,
1879                          struct fm_action *action_in,
1880                          uint32_t group,
1881                          uint8_t ingress,
1882                          struct rte_flow_error *error)
1883 {
1884         struct enic_fm_counter *ctr;
1885         struct fm_action *fma;
1886         uint64_t action_h;
1887         uint64_t entry_h;
1888         u64 args[3];
1889         int ret;
1890
1891         ENICPMD_FUNC_TRACE();
1892         /* Allocate action. */
1893         fma = &fm->cmd.va->fm_action;
1894         memcpy(fma, action_in, sizeof(*fma));
1895         args[0] = FM_ACTION_ALLOC;
1896         args[1] = fm->cmd.pa;
1897         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1898         if (ret != 0) {
1899                 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
1900                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1901                         NULL, "enic: devcmd(action-alloc)");
1902                 return ret;
1903         }
1904         action_h = args[0];
1905         fm_flow->action_handle = action_h;
1906         match_in->ftm_action = action_h;
1907         ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
1908
1909         /* Allocate counter if requested. */
1910         if (match_in->ftm_flags & FMEF_COUNTER) {
1911                 ret = enic_fm_counter_alloc(fm, error, &ctr);
1912                 if (ret) /* error has been filled in */
1913                         return ret;
1914                 fm_flow->counter_valid = true;
1915                 fm_flow->counter = ctr;
1916                 match_in->ftm_counter = ctr->handle;
1917         }
1918
1919         /*
1920          * Get the group's table (either TCAM or exact match table) and
1921          * add entry to it. If we use the exact match table, the handler
1922          * will translate the TCAM entry (match_in) to the appropriate
1923          * exact match entry and use that instead.
1924          */
1925         entry_h = FM_INVALID_HANDLE;
1926         if (group == FM_TCAM_RTE_GROUP) {
1927                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
1928                                              error);
1929                 if (ret)
1930                         return ret;
1931                 /* Jump action might have a ref to fet */
1932                 fm_flow->fet = fm->fet;
1933                 fm->fet = NULL;
1934         } else {
1935                 struct enic_fm_fet *fet = NULL;
1936
1937                 ret = enic_fet_get(fm, group, ingress,
1938                                    &match_in->ftm_mask, &fet, error);
1939                 if (ret)
1940                         return ret;
1941                 fm_flow->fet = fet;
1942                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
1943                                               error);
1944                 if (ret)
1945                         return ret;
1946         }
1947         /* Clear counter after adding entry, as it requires in-use counter */
1948         if (fm_flow->counter_valid) {
1949                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
1950                 if (ret)
1951                         return ret;
1952         }
1953         fm_flow->entry_handle = entry_h;
1954         return 0;
1955 }
1956
1957 /* Push match-action to the NIC. */
1958 static struct rte_flow *
1959 enic_fm_flow_add_entry(struct enic_flowman *fm,
1960                        struct fm_tcam_match_entry *match_in,
1961                        struct fm_action *action_in,
1962                        const struct rte_flow_attr *attrs,
1963                        struct rte_flow_error *error)
1964 {
1965         struct enic_fm_flow *fm_flow;
1966         struct rte_flow *flow;
1967
1968         ENICPMD_FUNC_TRACE();
1969         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
1970         flow = calloc(1, sizeof(*flow));
1971         fm_flow = calloc(1, sizeof(*fm_flow));
1972         if (flow == NULL || fm_flow == NULL) {
1973                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1974                         NULL, "enic: cannot allocate rte_flow");
1975                 free(flow);
1976                 free(fm_flow);
1977                 return NULL;
1978         }
1979         flow->fm = fm_flow;
1980         fm_flow->action_handle = FM_INVALID_HANDLE;
1981         fm_flow->entry_handle = FM_INVALID_HANDLE;
1982         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
1983                                      attrs->group, attrs->ingress, error)) {
1984                 enic_fm_flow_free(fm, flow);
1985                 return NULL;
1986         }
1987         return flow;
1988 }
1989
1990 static void
1991 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
1992                    struct rte_flow_error *error)
1993 {
1994         struct enic_fm_flow *fm_flow;
1995         struct enic_fm_jump_flow *j;
1996         struct fm_action *fma;
1997         uint32_t group;
1998
1999         ENICPMD_FUNC_TRACE();
2000         /*
2001          * Find the saved flows that should jump to the new table (fet).
2002          * Then delete the old TCAM entry that jumps to the default table,
2003          * and add a new one that jumps to the new table.
2004          */
2005         group = fet->group;
2006         j = find_jump_flow(fm, group);
2007         while (j) {
2008                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2009                             j->flow, group);
2010                 /* Delete old entry */
2011                 fm_flow = j->flow->fm;
2012                 __enic_fm_flow_free(fm, fm_flow);
2013
2014                 /* Add new entry */
2015                 fma = &j->action;
2016                 fma->fma_action_ops[0].exact.handle = fet->handle;
2017                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2018                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2019                         /* Cannot roll back changes at the moment */
2020                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2021                                     j->flow);
2022                 } else {
2023                         fm_flow->fet = fet;
2024                         fet->ref++;
2025                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2026                                     fet->group, fet->ref);
2027                 }
2028
2029                 TAILQ_REMOVE(&fm->jump_list, j, list);
2030                 free(j);
2031                 j = find_jump_flow(fm, group);
2032         }
2033 }
2034
2035 static void
2036 enic_fm_open_scratch(struct enic_flowman *fm)
2037 {
2038         fm->action_op_count = 0;
2039         fm->fet = NULL;
2040         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2041         memset(&fm->action, 0, sizeof(fm->action));
2042 }
2043
2044 static void
2045 enic_fm_close_scratch(struct enic_flowman *fm)
2046 {
2047         if (fm->fet) {
2048                 enic_fet_put(fm, fm->fet);
2049                 fm->fet = NULL;
2050         }
2051         fm->action_op_count = 0;
2052 }
2053
2054 static int
2055 enic_fm_flow_validate(struct rte_eth_dev *dev,
2056                       const struct rte_flow_attr *attrs,
2057                       const struct rte_flow_item pattern[],
2058                       const struct rte_flow_action actions[],
2059                       struct rte_flow_error *error)
2060 {
2061         struct fm_tcam_match_entry *fm_tcam_entry;
2062         struct fm_action *fm_action;
2063         struct enic_flowman *fm;
2064         int ret;
2065
2066         ENICPMD_FUNC_TRACE();
2067         fm = pmd_priv(dev)->fm;
2068         if (fm == NULL)
2069                 return -ENOTSUP;
2070         enic_fm_open_scratch(fm);
2071         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2072         if (!ret) {
2073                 fm_tcam_entry = &fm->tcam_entry;
2074                 fm_action = &fm->action;
2075                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2076                                         attrs->ingress);
2077         }
2078         enic_fm_close_scratch(fm);
2079         return ret;
2080 }
2081
2082 static int
2083 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2084                          struct rte_flow *flow, void *data,
2085                          struct rte_flow_error *error)
2086 {
2087         struct rte_flow_query_count *query;
2088         struct enic_fm_flow *fm_flow;
2089         struct enic *enic;
2090         u64 args[3];
2091         int rc;
2092
2093         ENICPMD_FUNC_TRACE();
2094         enic = pmd_priv(dev);
2095         query = data;
2096         fm_flow = flow->fm;
2097         if (!fm_flow->counter_valid)
2098                 return rte_flow_error_set(error, ENOTSUP,
2099                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2100                         "enic: flow does not have counter");
2101
2102         args[0] = FM_COUNTER_QUERY;
2103         args[1] = fm_flow->counter->handle;
2104         args[2] = query->reset;
2105         rc = vnic_dev_flowman_cmd(enic->vdev, args, 3);
2106         if (rc) {
2107                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2108                             rc, fm_flow->counter->handle);
2109                 return rc;
2110         }
2111         query->hits_set = 1;
2112         query->hits = args[0];
2113         query->bytes_set = 1;
2114         query->bytes = args[1];
2115         return 0;
2116 }
2117
2118 static int
2119 enic_fm_flow_query(struct rte_eth_dev *dev,
2120                    struct rte_flow *flow,
2121                    const struct rte_flow_action *actions,
2122                    void *data,
2123                    struct rte_flow_error *error)
2124 {
2125         int ret = 0;
2126
2127         ENICPMD_FUNC_TRACE();
2128         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2129                 switch (actions->type) {
2130                 case RTE_FLOW_ACTION_TYPE_VOID:
2131                         break;
2132                 case RTE_FLOW_ACTION_TYPE_COUNT:
2133                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2134                         break;
2135                 default:
2136                         return rte_flow_error_set(error, ENOTSUP,
2137                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2138                                                   actions,
2139                                                   "action not supported");
2140                 }
2141                 if (ret < 0)
2142                         return ret;
2143         }
2144         return 0;
2145 }
2146
2147 static struct rte_flow *
2148 enic_fm_flow_create(struct rte_eth_dev *dev,
2149                     const struct rte_flow_attr *attrs,
2150                     const struct rte_flow_item pattern[],
2151                     const struct rte_flow_action actions[],
2152                     struct rte_flow_error *error)
2153 {
2154         struct fm_tcam_match_entry *fm_tcam_entry;
2155         struct fm_action *fm_action;
2156         struct enic_flowman *fm;
2157         struct enic_fm_fet *fet;
2158         struct rte_flow *flow;
2159         struct enic *enic;
2160         int ret;
2161
2162         ENICPMD_FUNC_TRACE();
2163         enic = pmd_priv(dev);
2164         fm = enic->fm;
2165         if (fm == NULL) {
2166                 rte_flow_error_set(error, ENOTSUP,
2167                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2168                         "flowman is not initialized");
2169                 return NULL;
2170         }
2171         enic_fm_open_scratch(fm);
2172         flow = NULL;
2173         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2174         if (ret < 0)
2175                 goto error_with_scratch;
2176         fm_tcam_entry = &fm->tcam_entry;
2177         fm_action = &fm->action;
2178         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2179                                       attrs, error);
2180         if (flow) {
2181                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2182                 fet = flow->fm->fet;
2183                 if (fet && fet->default_key) {
2184                         /*
2185                          * Jump to non-existent group? Save the relevant info
2186                          * so we can convert this flow when that group
2187                          * materializes.
2188                          */
2189                         save_jump_flow(fm, flow, fet->group,
2190                                        fm_tcam_entry, fm_action);
2191                 } else if (fet && fet->ref == 1) {
2192                         /*
2193                          * A new table is created. Convert the saved flows
2194                          * that should jump to this group.
2195                          */
2196                         convert_jump_flows(fm, fet, error);
2197                 }
2198         }
2199
2200 error_with_scratch:
2201         enic_fm_close_scratch(fm);
2202         return flow;
2203 }
2204
2205 static int
2206 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2207                      __rte_unused struct rte_flow_error *error)
2208 {
2209         struct enic *enic = pmd_priv(dev);
2210
2211         ENICPMD_FUNC_TRACE();
2212         if (enic->fm == NULL)
2213                 return 0;
2214         LIST_REMOVE(flow, next);
2215         enic_fm_flow_free(enic->fm, flow);
2216         return 0;
2217 }
2218
2219 static int
2220 enic_fm_flow_flush(struct rte_eth_dev *dev,
2221                    __rte_unused struct rte_flow_error *error)
2222 {
2223         struct enic_fm_flow *fm_flow;
2224         struct enic_flowman *fm;
2225         struct rte_flow *flow;
2226         struct enic *enic = pmd_priv(dev);
2227
2228         ENICPMD_FUNC_TRACE();
2229         if (enic->fm == NULL)
2230                 return 0;
2231         fm = enic->fm;
2232         while (!LIST_EMPTY(&enic->flows)) {
2233                 flow = LIST_FIRST(&enic->flows);
2234                 fm_flow = flow->fm;
2235                 LIST_REMOVE(flow, next);
2236                 /*
2237                  * If tables are null, then vNIC is closing, and the firmware
2238                  * has already cleaned up flowman state. So do not try to free
2239                  * resources, as it only causes errors.
2240                  */
2241                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2242                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2243                         fm_flow->action_handle = FM_INVALID_HANDLE;
2244                         fm_flow->fet = NULL;
2245                 }
2246                 enic_fm_flow_free(fm, flow);
2247         }
2248         return 0;
2249 }
2250
2251 static int
2252 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2253 {
2254         u64 args[2];
2255         int rc;
2256
2257         args[0] = FM_MATCH_TABLE_FREE;
2258         args[1] = handle;
2259         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
2260         if (rc)
2261                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2262                             rc, handle);
2263         return rc;
2264 }
2265
2266 static int
2267 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2268                         uint32_t max_entries, uint64_t *handle)
2269 {
2270         struct fm_tcam_match_table *tcam_tbl;
2271         struct enic *enic;
2272         u64 args[2];
2273         int rc;
2274
2275         ENICPMD_FUNC_TRACE();
2276         enic = fm->enic;
2277         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2278         tcam_tbl->ftt_direction = direction;
2279         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2280         tcam_tbl->ftt_max_entries = max_entries;
2281         args[0] = FM_TCAM_TABLE_ALLOC;
2282         args[1] = fm->cmd.pa;
2283         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2284         if (rc) {
2285                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2286                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2287                 return rc;
2288         }
2289         *handle = args[0];
2290         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2291                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2292         return 0;
2293 }
2294
2295 static int
2296 enic_fm_init_counters(struct enic_flowman *fm)
2297 {
2298         ENICPMD_FUNC_TRACE();
2299         SLIST_INIT(&fm->counters);
2300         return enic_fm_more_counters(fm);
2301 }
2302
2303 static void
2304 enic_fm_free_all_counters(struct enic_flowman *fm)
2305 {
2306         struct enic *enic;
2307         u64 args[2];
2308         int rc;
2309
2310         enic = fm->enic;
2311         args[0] = FM_COUNTER_BRK;
2312         args[1] = 0;
2313         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2314         if (rc != 0)
2315                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2316         rte_free(fm->counter_stack);
2317 }
2318
2319 static int
2320 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2321 {
2322         int rc;
2323
2324         ENICPMD_FUNC_TRACE();
2325         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2326                                     &fm->ig_tcam_hndl);
2327         if (rc)
2328                 return rc;
2329         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2330                                     &fm->eg_tcam_hndl);
2331         return rc;
2332 }
2333
2334 static void
2335 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2336 {
2337         ENICPMD_FUNC_TRACE();
2338         if (fm->ig_tcam_hndl) {
2339                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2340                             fm->ig_tcam_hndl);
2341                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2342                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2343         }
2344         if (fm->eg_tcam_hndl) {
2345                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2346                             fm->eg_tcam_hndl);
2347                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2348                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2349         }
2350 }
2351
2352 int
2353 enic_fm_init(struct enic *enic)
2354 {
2355         struct enic_flowman *fm;
2356         u8 name[NAME_MAX];
2357         int rc;
2358
2359         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2360                 return 0;
2361         ENICPMD_FUNC_TRACE();
2362         fm = calloc(1, sizeof(*fm));
2363         if (fm == NULL) {
2364                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2365                 return -ENOMEM;
2366         }
2367         fm->enic = enic;
2368         TAILQ_INIT(&fm->fet_list);
2369         TAILQ_INIT(&fm->jump_list);
2370         /* Allocate host memory for flowman commands */
2371         snprintf((char *)name, NAME_MAX, "fm-cmd-%s", enic->bdf_name);
2372         fm->cmd.va = enic_alloc_consistent(enic,
2373                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2374         if (!fm->cmd.va) {
2375                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2376                 rc = -ENOMEM;
2377                 goto error_fm;
2378         }
2379         /* Allocate TCAM tables upfront as they are the main tables */
2380         rc = enic_fm_alloc_tcam_tables(fm);
2381         if (rc) {
2382                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2383                 goto error_cmd;
2384         }
2385         /* Then a number of counters */
2386         rc = enic_fm_init_counters(fm);
2387         if (rc) {
2388                 ENICPMD_LOG(ERR, "cannot alloc counters");
2389                 goto error_tables;
2390         }
2391         /*
2392          * One default exact match table for each direction. We hold onto
2393          * it until close.
2394          */
2395         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2396         if (rc) {
2397                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2398                 goto error_counters;
2399         }
2400         fm->default_ig_fet->ref = 1;
2401         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2402         if (rc) {
2403                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2404                 goto error_ig_fet;
2405         }
2406         fm->default_eg_fet->ref = 1;
2407         enic->fm = fm;
2408         return 0;
2409
2410 error_ig_fet:
2411         enic_fet_free(fm, fm->default_ig_fet);
2412 error_counters:
2413         enic_fm_free_all_counters(fm);
2414 error_tables:
2415         enic_fm_free_tcam_tables(fm);
2416 error_cmd:
2417         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2418                 fm->cmd.va, fm->cmd.pa);
2419 error_fm:
2420         free(fm);
2421         return rc;
2422 }
2423
2424 void
2425 enic_fm_destroy(struct enic *enic)
2426 {
2427         struct enic_flowman *fm;
2428         struct enic_fm_fet *fet;
2429
2430         if (enic->fm == NULL)
2431                 return;
2432         ENICPMD_FUNC_TRACE();
2433         fm = enic->fm;
2434         enic_fet_free(fm, fm->default_eg_fet);
2435         enic_fet_free(fm, fm->default_ig_fet);
2436         /* Free all exact match tables still open */
2437         while (!TAILQ_EMPTY(&fm->fet_list)) {
2438                 fet = TAILQ_FIRST(&fm->fet_list);
2439                 enic_fet_free(fm, fet);
2440         }
2441         enic_fm_free_tcam_tables(fm);
2442         enic_fm_free_all_counters(fm);
2443         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2444                 fm->cmd.va, fm->cmd.pa);
2445         fm->cmd.va = NULL;
2446         free(fm);
2447         enic->fm = NULL;
2448 }
2449
2450 const struct rte_flow_ops enic_fm_flow_ops = {
2451         .validate = enic_fm_flow_validate,
2452         .create = enic_fm_flow_create,
2453         .destroy = enic_fm_flow_destroy,
2454         .flush = enic_fm_flow_flush,
2455         .query = enic_fm_flow_query,
2456 };