drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_geneve.h>
  12 #include <rte_hash.h>
  13 #include <rte_jhash.h>
  14 #include <rte_ip.h>
  15 #include <rte_udp.h>
  16 #include <rte_memzone.h>
  17
  18 #include "enic_compat.h"
  19 #include "enic.h"
  20 #include "vnic_dev.h"
  21 #include "vnic_nic.h"
  22
  23 #define IP_DEFTTL  64   /* from RFC 1340. */
  24 #define IP6_VTC_FLOW 0x60000000
  25
  26 /* Up to 1024 TCAM entries */
  27 #define FM_MAX_TCAM_TABLE_SIZE 1024
  28
  29 /* Up to 4096 entries per exact match table */
  30 #define FM_MAX_EXACT_TABLE_SIZE 4096
  31
  32 /* Number of counters to increase on for each increment */
  33 #define FM_COUNTERS_EXPAND  100
  34
  35 #define FM_INVALID_HANDLE 0
  36
  37 /* Low priority used for implicit VF -> representor flow */
  38 #define FM_LOWEST_PRIORITY 100000
  39
  40 /* High priority used for implicit representor -> VF flow */
  41 #define FM_HIGHEST_PRIORITY 0
  42
  43 /* Tag used for implicit VF <-> representor flows */
  44 #define FM_VF_REP_TAG 1
  45
  46 /* Max number of actions supported by VIC is 2K. Make hash table double that. */
  47 #define FM_MAX_ACTION_TABLE_SIZE 4096
  48
  49 /*
  50  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  51  * Use a simple scheme to map groups to tables.
  52  * Group 0 uses the single TCAM tables, one for each direction.
  53  * Group 1, 2, ... uses its own exact match table.
  54  *
  55  * The TCAM tables are allocated upfront during init.
  56  *
  57  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  58  *
  59  * 1. Add a flow that jumps from group 0 to group N.
  60  *
  61  * If N does not exist, we allocate an exact match table for it, using
  62  * a dummy key. A key is required for the table.
  63  *
  64  * 2. Add a flow that uses group N.
  65  *
  66  * If N does not exist, we allocate an exact match table for it, using
  67  * the flow's key. Subsequent flows to the same group all should have
  68  * the same key.
  69  *
  70  * Without a jump flow to N, N is not reachable in hardware. No packets
  71  * reach N and match.
  72  *
  73  * 3. Add a flow to an empty group N.
  74  *
  75  * N has been created via (1) and the dummy key. We free that table, allocate
  76  * a new table using the new flow's key. Also re-do the existing jump flow to
  77  * point to the new table.
  78  */
  79 #define FM_TCAM_RTE_GROUP 0
  80
  81 struct enic_fm_fet {
  82         TAILQ_ENTRY(enic_fm_fet) list;
  83         uint32_t group; /* rte_flow group ID */
  84         uint64_t handle; /* Exact match table handle from flowman */
  85         uint8_t ingress;
  86         uint8_t default_key;
  87         int ref; /* Reference count via get/put */
  88         struct fm_key_template key; /* Key associated with the table */
  89 };
  90
  91 struct enic_fm_counter {
  92         SLIST_ENTRY(enic_fm_counter) next;
  93         uint32_t handle;
  94 };
  95
  96 struct enic_fm_action {
  97         int ref;
  98         uint64_t handle;
  99         struct fm_action key;
 100 };
 101
 102 /* rte_flow.fm */
 103 struct enic_fm_flow {
 104         bool counter_valid;
 105         uint64_t entry_handle;
 106         struct enic_fm_action  *action;
 107         struct enic_fm_counter *counter;
 108         struct enic_fm_fet *fet;
 109         /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
 110         struct enic_fm_flow *hairpin_steer_flow;
 111 };
 112
 113 struct enic_fm_jump_flow {
 114         TAILQ_ENTRY(enic_fm_jump_flow) list;
 115         struct rte_flow *flow;
 116         uint32_t group;
 117         struct fm_tcam_match_entry match;
 118         struct fm_action action;
 119 };
 120
 121 /*
 122  * Flowman uses host memory for commands. This structure is allocated
 123  * in DMA-able memory.
 124  */
 125 union enic_flowman_cmd_mem {
 126         struct fm_tcam_match_table fm_tcam_match_table;
 127         struct fm_exact_match_table fm_exact_match_table;
 128         struct fm_tcam_match_entry fm_tcam_match_entry;
 129         struct fm_exact_match_entry fm_exact_match_entry;
 130         struct fm_action fm_action;
 131 };
 132
 133 /*
 134  * PF has a flowman instance, and VF representors share it with PF.
 135  * PF allocates this structure and owns it. VF representors borrow
 136  * the PF's structure during API calls (e.g. create, query).
 137  */
 138 struct enic_flowman {
 139         struct enic *owner_enic; /* PF */
 140         struct enic *user_enic;  /* API caller (PF or representor) */
 141         /*
 142          * Representors and PF share the same underlying flowman.
 143          * Lock API calls to serialize accesses from them. Only used
 144          * when VF representors are present.
 145          */
 146         rte_spinlock_t lock;
 147         /* Command buffer */
 148         struct {
 149                 union enic_flowman_cmd_mem *va;
 150                 dma_addr_t pa;
 151         } cmd;
 152         /* TCAM tables allocated upfront, used for group 0 */
 153         uint64_t ig_tcam_hndl;
 154         uint64_t eg_tcam_hndl;
 155         /* Counters */
 156         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 157         void *counter_stack;
 158         uint32_t counters_alloced;
 159         /* Exact match tables for groups != 0, dynamically allocated */
 160         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 161         /*
 162          * Default exact match tables used for jump actions to
 163          * non-existent groups.
 164          */
 165         struct enic_fm_fet *default_eg_fet;
 166         struct enic_fm_fet *default_ig_fet;
 167         /* hash table for Action reuse */
 168         struct rte_hash *action_hash;
 169         /* Flows that jump to the default table above */
 170         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 171         /*
 172          * Scratch data used during each invocation of flow_create
 173          * and flow_validate.
 174          */
 175         struct enic_fm_fet *fet;
 176         struct fm_tcam_match_entry tcam_entry;
 177         struct fm_action action;
 178         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 179         int action_op_count;
 180         /* Tags used for representor flows */
 181         uint8_t vf_rep_tag;
 182         /* For auto-added steer action for hairpin */
 183         int need_hairpin_steer;
 184         uint64_t hairpin_steer_vnic_h;
 185 };
 186
 187 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 188 /*
 189  * API functions (create, destroy, validate, flush) call begin_fm()
 190  * upon entering to save the caller enic (PF or VF representor) and
 191  * lock. Upon exit, they call end_fm() to unlock.
 192  */
 193 static struct enic_flowman *begin_fm(struct enic *enic);
 194 static void end_fm(struct enic_flowman *fm);
 195 /* Delete internal flows created for representor paths */
 196 static void delete_rep_flows(struct enic *enic);
 197
 198 /*
 199  * Common arguments passed to copy_item functions. Use this structure
 200  * so we can easily add new arguments.
 201  * item: Item specification.
 202  * fm_tcam_entry: Flowman TCAM match entry.
 203  * header_level: 0 for outer header, 1 for inner header.
 204  */
 205 struct copy_item_args {
 206         const struct rte_flow_item *item;
 207         struct fm_tcam_match_entry *fm_tcam_entry;
 208         uint8_t header_level;
 209         struct rte_flow_error *error;
 210 };
 211
 212 /* functions for copying items into flowman match */
 213 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 214
 215 /* Info about how to copy items into flowman match */
 216 struct enic_fm_items {
 217         /* Function for copying and validating an item. */
 218         enic_copy_item_fn * const copy_item;
 219         /* List of valid previous items. */
 220         const enum rte_flow_item_type * const prev_items;
 221         /*
 222          * True if it's OK for this item to be the first item. For some NIC
 223          * versions, it's invalid to start the stack above layer 3.
 224          */
 225         const uint8_t valid_start_item;
 226 };
 227
 228 static enic_copy_item_fn enic_fm_copy_item_eth;
 229 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 230 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 231 static enic_copy_item_fn enic_fm_copy_item_raw;
 232 static enic_copy_item_fn enic_fm_copy_item_sctp;
 233 static enic_copy_item_fn enic_fm_copy_item_tcp;
 234 static enic_copy_item_fn enic_fm_copy_item_udp;
 235 static enic_copy_item_fn enic_fm_copy_item_vlan;
 236 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 237 static enic_copy_item_fn enic_fm_copy_item_gtp;
 238 static enic_copy_item_fn enic_fm_copy_item_geneve;
 239 static enic_copy_item_fn enic_fm_copy_item_geneve_opt;
 240 static enic_copy_item_fn enic_fm_copy_item_ecpri;
 241
 242 /* Ingress actions */
 243 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 244         RTE_FLOW_ACTION_TYPE_COUNT,
 245         RTE_FLOW_ACTION_TYPE_DROP,
 246         RTE_FLOW_ACTION_TYPE_FLAG,
 247         RTE_FLOW_ACTION_TYPE_JUMP,
 248         RTE_FLOW_ACTION_TYPE_MARK,
 249         RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
 250         RTE_FLOW_ACTION_TYPE_PORT_ID,
 251         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 252         RTE_FLOW_ACTION_TYPE_QUEUE,
 253         RTE_FLOW_ACTION_TYPE_RSS,
 254         RTE_FLOW_ACTION_TYPE_VOID,
 255         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 256         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 257         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 258 };
 259
 260 /* Egress actions */
 261 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 262         RTE_FLOW_ACTION_TYPE_COUNT,
 263         RTE_FLOW_ACTION_TYPE_DROP,
 264         RTE_FLOW_ACTION_TYPE_JUMP,
 265         RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
 266         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
 267         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
 268         RTE_FLOW_ACTION_TYPE_PORT_ID,
 269         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 270         RTE_FLOW_ACTION_TYPE_VOID,
 271         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 272         RTE_FLOW_ACTION_TYPE_END,
 273 };
 274
 275 static const struct enic_fm_items enic_fm_items[] = {
 276         [RTE_FLOW_ITEM_TYPE_RAW] = {
 277                 .copy_item = enic_fm_copy_item_raw,
 278                 .valid_start_item = 0,
 279                 .prev_items = (const enum rte_flow_item_type[]) {
 280                                RTE_FLOW_ITEM_TYPE_UDP,
 281                                RTE_FLOW_ITEM_TYPE_END,
 282                 },
 283         },
 284         [RTE_FLOW_ITEM_TYPE_ETH] = {
 285                 .copy_item = enic_fm_copy_item_eth,
 286                 .valid_start_item = 1,
 287                 .prev_items = (const enum rte_flow_item_type[]) {
 288                                RTE_FLOW_ITEM_TYPE_END,
 289                 },
 290         },
 291         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 292                 .copy_item = enic_fm_copy_item_vlan,
 293                 .valid_start_item = 1,
 294                 .prev_items = (const enum rte_flow_item_type[]) {
 295                                RTE_FLOW_ITEM_TYPE_ETH,
 296                                RTE_FLOW_ITEM_TYPE_END,
 297                 },
 298         },
 299         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 300                 .copy_item = enic_fm_copy_item_ipv4,
 301                 .valid_start_item = 1,
 302                 .prev_items = (const enum rte_flow_item_type[]) {
 303                                RTE_FLOW_ITEM_TYPE_ETH,
 304                                RTE_FLOW_ITEM_TYPE_VLAN,
 305                                RTE_FLOW_ITEM_TYPE_END,
 306                 },
 307         },
 308         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 309                 .copy_item = enic_fm_copy_item_ipv6,
 310                 .valid_start_item = 1,
 311                 .prev_items = (const enum rte_flow_item_type[]) {
 312                                RTE_FLOW_ITEM_TYPE_ETH,
 313                                RTE_FLOW_ITEM_TYPE_VLAN,
 314                                RTE_FLOW_ITEM_TYPE_END,
 315                 },
 316         },
 317         [RTE_FLOW_ITEM_TYPE_UDP] = {
 318                 .copy_item = enic_fm_copy_item_udp,
 319                 .valid_start_item = 1,
 320                 .prev_items = (const enum rte_flow_item_type[]) {
 321                                RTE_FLOW_ITEM_TYPE_IPV4,
 322                                RTE_FLOW_ITEM_TYPE_IPV6,
 323                                RTE_FLOW_ITEM_TYPE_END,
 324                 },
 325         },
 326         [RTE_FLOW_ITEM_TYPE_TCP] = {
 327                 .copy_item = enic_fm_copy_item_tcp,
 328                 .valid_start_item = 1,
 329                 .prev_items = (const enum rte_flow_item_type[]) {
 330                                RTE_FLOW_ITEM_TYPE_IPV4,
 331                                RTE_FLOW_ITEM_TYPE_IPV6,
 332                                RTE_FLOW_ITEM_TYPE_END,
 333                 },
 334         },
 335         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 336                 .copy_item = enic_fm_copy_item_sctp,
 337                 .valid_start_item = 0,
 338                 .prev_items = (const enum rte_flow_item_type[]) {
 339                                RTE_FLOW_ITEM_TYPE_IPV4,
 340                                RTE_FLOW_ITEM_TYPE_IPV6,
 341                                RTE_FLOW_ITEM_TYPE_END,
 342                 },
 343         },
 344         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 345                 .copy_item = enic_fm_copy_item_vxlan,
 346                 .valid_start_item = 1,
 347                 .prev_items = (const enum rte_flow_item_type[]) {
 348                                RTE_FLOW_ITEM_TYPE_UDP,
 349                                RTE_FLOW_ITEM_TYPE_END,
 350                 },
 351         },
 352         [RTE_FLOW_ITEM_TYPE_GTP] = {
 353                 .copy_item = enic_fm_copy_item_gtp,
 354                 .valid_start_item = 0,
 355                 .prev_items = (const enum rte_flow_item_type[]) {
 356                                RTE_FLOW_ITEM_TYPE_UDP,
 357                                RTE_FLOW_ITEM_TYPE_END,
 358                 },
 359         },
 360         [RTE_FLOW_ITEM_TYPE_GTPC] = {
 361                 .copy_item = enic_fm_copy_item_gtp,
 362                 .valid_start_item = 1,
 363                 .prev_items = (const enum rte_flow_item_type[]) {
 364                                RTE_FLOW_ITEM_TYPE_UDP,
 365                                RTE_FLOW_ITEM_TYPE_END,
 366                 },
 367         },
 368         [RTE_FLOW_ITEM_TYPE_GTPU] = {
 369                 .copy_item = enic_fm_copy_item_gtp,
 370                 .valid_start_item = 1,
 371                 .prev_items = (const enum rte_flow_item_type[]) {
 372                                RTE_FLOW_ITEM_TYPE_UDP,
 373                                RTE_FLOW_ITEM_TYPE_END,
 374                 },
 375         },
 376         [RTE_FLOW_ITEM_TYPE_GENEVE] = {
 377                 .copy_item = enic_fm_copy_item_geneve,
 378                 .valid_start_item = 1,
 379                 .prev_items = (const enum rte_flow_item_type[]) {
 380                                RTE_FLOW_ITEM_TYPE_ETH,
 381                                RTE_FLOW_ITEM_TYPE_IPV4,
 382                                RTE_FLOW_ITEM_TYPE_IPV6,
 383                                RTE_FLOW_ITEM_TYPE_UDP,
 384                                RTE_FLOW_ITEM_TYPE_END,
 385                 },
 386         },
 387         [RTE_FLOW_ITEM_TYPE_GENEVE_OPT] = {
 388                 .copy_item = enic_fm_copy_item_geneve_opt,
 389                 .valid_start_item = 1,
 390                 /* Can match at most 1 option */
 391                 .prev_items = (const enum rte_flow_item_type[]) {
 392                                RTE_FLOW_ITEM_TYPE_GENEVE,
 393                                RTE_FLOW_ITEM_TYPE_END,
 394                 },
 395         },
 396         [RTE_FLOW_ITEM_TYPE_ECPRI] = {
 397                 .copy_item = enic_fm_copy_item_ecpri,
 398                 .valid_start_item = 1,
 399                 .prev_items = (const enum rte_flow_item_type[]) {
 400                                RTE_FLOW_ITEM_TYPE_ETH,
 401                                RTE_FLOW_ITEM_TYPE_UDP,
 402                                RTE_FLOW_ITEM_TYPE_END,
 403                 },
 404         },
 405 };
 406
 407 static int
 408 enic_fm_copy_item_eth(struct copy_item_args *arg)
 409 {
 410         const struct rte_flow_item *item = arg->item;
 411         const struct rte_flow_item_eth *spec = item->spec;
 412         const struct rte_flow_item_eth *mask = item->mask;
 413         const uint8_t lvl = arg->header_level;
 414         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 415         struct fm_header_set *fm_data, *fm_mask;
 416
 417         ENICPMD_FUNC_TRACE();
 418         /* Match all if no spec */
 419         if (!spec)
 420                 return 0;
 421         if (!mask)
 422                 mask = &rte_flow_item_eth_mask;
 423         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 424         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 425         fm_data->fk_header_select |= FKH_ETHER;
 426         fm_mask->fk_header_select |= FKH_ETHER;
 427         memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
 428         memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
 429         return 0;
 430 }
 431
 432 static int
 433 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 434 {
 435         const struct rte_flow_item *item = arg->item;
 436         const struct rte_flow_item_vlan *spec = item->spec;
 437         const struct rte_flow_item_vlan *mask = item->mask;
 438         const uint8_t lvl = arg->header_level;
 439         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 440         struct fm_header_set *fm_data, *fm_mask;
 441         struct rte_ether_hdr *eth_mask;
 442         struct rte_ether_hdr *eth_val;
 443         uint32_t meta;
 444
 445         ENICPMD_FUNC_TRACE();
 446         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 447         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 448         /* Outer and inner packet vlans need different flags */
 449         meta = FKM_VLAN_PRES;
 450         if (lvl > 0)
 451                 meta = FKM_QTAG;
 452         fm_data->fk_metadata |= meta;
 453         fm_mask->fk_metadata |= meta;
 454
 455         /* Match all if no spec */
 456         if (!spec)
 457                 return 0;
 458         if (!mask)
 459                 mask = &rte_flow_item_vlan_mask;
 460
 461         eth_mask = (void *)&fm_mask->l2.eth;
 462         eth_val = (void *)&fm_data->l2.eth;
 463
 464         /*
 465          * Outer TPID cannot be matched. If inner_type is 0, use what is
 466          * in the eth header.
 467          */
 468         if (eth_mask->ether_type && mask->inner_type)
 469                 return -ENOTSUP;
 470
 471         /*
 472          * When packet matching, the VIC always compares vlan-stripped
 473          * L2, regardless of vlan stripping settings. So, the inner type
 474          * from vlan becomes the ether type of the eth header.
 475          */
 476         if (mask->inner_type) {
 477                 eth_mask->ether_type = mask->inner_type;
 478                 eth_val->ether_type = spec->inner_type;
 479         }
 480         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 481         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 482         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 483         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 484         return 0;
 485 }
 486
 487 static int
 488 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 489 {
 490         const struct rte_flow_item *item = arg->item;
 491         const struct rte_flow_item_ipv4 *spec = item->spec;
 492         const struct rte_flow_item_ipv4 *mask = item->mask;
 493         const uint8_t lvl = arg->header_level;
 494         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 495         struct fm_header_set *fm_data, *fm_mask;
 496
 497         ENICPMD_FUNC_TRACE();
 498         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 499         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 500         fm_data->fk_metadata |= FKM_IPV4;
 501         fm_mask->fk_metadata |= FKM_IPV4;
 502
 503         if (!spec)
 504                 return 0;
 505         if (!mask)
 506                 mask = &rte_flow_item_ipv4_mask;
 507
 508         fm_data->fk_header_select |= FKH_IPV4;
 509         fm_mask->fk_header_select |= FKH_IPV4;
 510         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 511         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 512         return 0;
 513 }
 514
 515 static int
 516 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 517 {
 518         const struct rte_flow_item *item = arg->item;
 519         const struct rte_flow_item_ipv6 *spec = item->spec;
 520         const struct rte_flow_item_ipv6 *mask = item->mask;
 521         const uint8_t lvl = arg->header_level;
 522         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 523         struct fm_header_set *fm_data, *fm_mask;
 524
 525         ENICPMD_FUNC_TRACE();
 526         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 527         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 528         fm_data->fk_metadata |= FKM_IPV6;
 529         fm_mask->fk_metadata |= FKM_IPV6;
 530
 531         if (!spec)
 532                 return 0;
 533         if (!mask)
 534                 mask = &rte_flow_item_ipv6_mask;
 535
 536         fm_data->fk_header_select |= FKH_IPV6;
 537         fm_mask->fk_header_select |= FKH_IPV6;
 538         memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
 539         memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
 540         return 0;
 541 }
 542
 543 static int
 544 enic_fm_copy_item_udp(struct copy_item_args *arg)
 545 {
 546         const struct rte_flow_item *item = arg->item;
 547         const struct rte_flow_item_udp *spec = item->spec;
 548         const struct rte_flow_item_udp *mask = item->mask;
 549         const uint8_t lvl = arg->header_level;
 550         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 551         struct fm_header_set *fm_data, *fm_mask;
 552
 553         ENICPMD_FUNC_TRACE();
 554         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 555         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 556         fm_data->fk_metadata |= FKM_UDP;
 557         fm_mask->fk_metadata |= FKM_UDP;
 558
 559         if (!spec)
 560                 return 0;
 561         if (!mask)
 562                 mask = &rte_flow_item_udp_mask;
 563
 564         fm_data->fk_header_select |= FKH_UDP;
 565         fm_mask->fk_header_select |= FKH_UDP;
 566         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 567         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 568         return 0;
 569 }
 570
 571 static int
 572 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 573 {
 574         const struct rte_flow_item *item = arg->item;
 575         const struct rte_flow_item_tcp *spec = item->spec;
 576         const struct rte_flow_item_tcp *mask = item->mask;
 577         const uint8_t lvl = arg->header_level;
 578         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 579         struct fm_header_set *fm_data, *fm_mask;
 580
 581         ENICPMD_FUNC_TRACE();
 582         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 583         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 584         fm_data->fk_metadata |= FKM_TCP;
 585         fm_mask->fk_metadata |= FKM_TCP;
 586
 587         if (!spec)
 588                 return 0;
 589         if (!mask)
 590                 mask = &rte_flow_item_tcp_mask;
 591
 592         fm_data->fk_header_select |= FKH_TCP;
 593         fm_mask->fk_header_select |= FKH_TCP;
 594         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 595         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 596         return 0;
 597 }
 598
 599 static int
 600 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 601 {
 602         const struct rte_flow_item *item = arg->item;
 603         const struct rte_flow_item_sctp *spec = item->spec;
 604         const struct rte_flow_item_sctp *mask = item->mask;
 605         const uint8_t lvl = arg->header_level;
 606         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 607         struct fm_header_set *fm_data, *fm_mask;
 608         uint8_t *ip_proto_mask = NULL;
 609         uint8_t *ip_proto = NULL;
 610         uint32_t l3_fkh;
 611
 612         ENICPMD_FUNC_TRACE();
 613         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 614         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 615         /*
 616          * The NIC filter API has no flags for "match sctp", so explicitly
 617          * set the protocol number in the IP pattern.
 618          */
 619         if (fm_data->fk_metadata & FKM_IPV4) {
 620                 struct rte_ipv4_hdr *ip;
 621                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 622                 ip_proto_mask = &ip->next_proto_id;
 623                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 624                 ip_proto = &ip->next_proto_id;
 625                 l3_fkh = FKH_IPV4;
 626         } else if (fm_data->fk_metadata & FKM_IPV6) {
 627                 struct rte_ipv6_hdr *ip;
 628                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 629                 ip_proto_mask = &ip->proto;
 630                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 631                 ip_proto = &ip->proto;
 632                 l3_fkh = FKH_IPV6;
 633         } else {
 634                 /* Need IPv4/IPv6 pattern first */
 635                 return -EINVAL;
 636         }
 637         *ip_proto = IPPROTO_SCTP;
 638         *ip_proto_mask = 0xff;
 639         fm_data->fk_header_select |= l3_fkh;
 640         fm_mask->fk_header_select |= l3_fkh;
 641
 642         if (!spec)
 643                 return 0;
 644         if (!mask)
 645                 mask = &rte_flow_item_sctp_mask;
 646
 647         fm_data->fk_header_select |= FKH_L4RAW;
 648         fm_mask->fk_header_select |= FKH_L4RAW;
 649         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 650         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 651         return 0;
 652 }
 653
 654 static int
 655 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 656 {
 657         const struct rte_flow_item *item = arg->item;
 658         const struct rte_flow_item_vxlan *spec = item->spec;
 659         const struct rte_flow_item_vxlan *mask = item->mask;
 660         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 661         struct fm_header_set *fm_data, *fm_mask;
 662
 663         ENICPMD_FUNC_TRACE();
 664         /* Only 2 header levels (outer and inner) allowed */
 665         if (arg->header_level > 0)
 666                 return -EINVAL;
 667
 668         fm_data = &entry->ftm_data.fk_hdrset[0];
 669         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 670         fm_data->fk_metadata |= FKM_VXLAN;
 671         fm_mask->fk_metadata |= FKM_VXLAN;
 672         /* items from here on out are inner header items */
 673         arg->header_level = 1;
 674
 675         /* Match all if no spec */
 676         if (!spec)
 677                 return 0;
 678         if (!mask)
 679                 mask = &rte_flow_item_vxlan_mask;
 680
 681         fm_data->fk_header_select |= FKH_VXLAN;
 682         fm_mask->fk_header_select |= FKH_VXLAN;
 683         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 684         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 685         return 0;
 686 }
 687
 688 static int
 689 enic_fm_copy_item_gtp(struct copy_item_args *arg)
 690 {
 691         const struct rte_flow_item *item = arg->item;
 692         const struct rte_flow_item_gtp *spec = item->spec;
 693         const struct rte_flow_item_gtp *mask = item->mask;
 694         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 695         struct fm_header_set *fm_data, *fm_mask;
 696         int off;
 697         uint16_t udp_gtp_uc_port_be = 0;
 698
 699         ENICPMD_FUNC_TRACE();
 700         /* Only 2 header levels (outer and inner) allowed */
 701         if (arg->header_level > 0)
 702                 return -EINVAL;
 703
 704         fm_data = &entry->ftm_data.fk_hdrset[0];
 705         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 706
 707         switch (item->type) {
 708         case RTE_FLOW_ITEM_TYPE_GTP:
 709         {
 710                 /* For vanilla GTP, the UDP destination port must be specified
 711                  * but value of the port is not enforced here.
 712                  */
 713                 if (!(fm_data->fk_metadata & FKM_UDP) ||
 714                     !(fm_data->fk_header_select & FKH_UDP) ||
 715                     fm_data->l4.udp.fk_dest == 0)
 716                         return -EINVAL;
 717                 if (!(fm_mask->fk_metadata & FKM_UDP) ||
 718                     !(fm_mask->fk_header_select & FKH_UDP) ||
 719                     fm_mask->l4.udp.fk_dest != 0xFFFF)
 720                         return -EINVAL;
 721                 break;
 722         }
 723         case RTE_FLOW_ITEM_TYPE_GTPC:
 724         {
 725                 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPC_UDP_PORT);
 726                 break;
 727         }
 728         case RTE_FLOW_ITEM_TYPE_GTPU:
 729         {
 730                 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPU_UDP_PORT);
 731                 break;
 732         }
 733         default:
 734                 RTE_ASSERT(0);
 735         }
 736
 737         /* The GTP-C or GTP-U UDP destination port must be matched. */
 738         if (udp_gtp_uc_port_be) {
 739                 if (fm_data->fk_metadata & FKM_UDP &&
 740                     fm_data->fk_header_select & FKH_UDP &&
 741                     fm_data->l4.udp.fk_dest != udp_gtp_uc_port_be)
 742                         return -EINVAL;
 743                 if (fm_mask->fk_metadata & FKM_UDP &&
 744                     fm_mask->fk_header_select & FKH_UDP &&
 745                     fm_mask->l4.udp.fk_dest != 0xFFFF)
 746                         return -EINVAL;
 747
 748                 /* In any case, add match for GTP-C GTP-U UDP dst port */
 749                 fm_data->fk_metadata |= FKM_UDP;
 750                 fm_data->fk_header_select |= FKH_UDP;
 751                 fm_data->l4.udp.fk_dest = udp_gtp_uc_port_be;
 752                 fm_mask->fk_metadata |= FKM_UDP;
 753                 fm_mask->fk_header_select |= FKH_UDP;
 754                 fm_mask->l4.udp.fk_dest = 0xFFFF;
 755         }
 756
 757         /* NIC does not support GTP tunnels. No Items are allowed after this.
 758          * This prevents the specification of further items.
 759          */
 760         arg->header_level = 0;
 761
 762         /* Match all if no spec */
 763         if (!spec)
 764                 return 0;
 765         if (!mask)
 766                 mask = &rte_flow_item_gtp_mask;
 767
 768         /*
 769          * Use the raw L4 buffer to match GTP as fm_header_set does not have
 770          * GTP header. UDP dst port must be specific. Using the raw buffer
 771          * does not affect such UDP item, since we skip UDP in the raw buffer.
 772          */
 773         fm_data->fk_header_select |= FKH_L4RAW;
 774         fm_mask->fk_header_select |= FKH_L4RAW;
 775         off = sizeof(fm_data->l4.udp);
 776         memcpy(&fm_data->l4.rawdata[off], spec, sizeof(*spec));
 777         memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(*mask));
 778         return 0;
 779 }
 780
 781 static int
 782 enic_fm_copy_item_geneve(struct copy_item_args *arg)
 783 {
 784         const struct rte_flow_item *item = arg->item;
 785         const struct rte_flow_item_geneve *spec = item->spec;
 786         const struct rte_flow_item_geneve *mask = item->mask;
 787         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 788         struct fm_header_set *fm_data, *fm_mask;
 789         int off;
 790
 791         ENICPMD_FUNC_TRACE();
 792         /* Only 2 header levels (outer and inner) allowed */
 793         if (arg->header_level > 0)
 794                 return -EINVAL;
 795
 796         fm_data = &entry->ftm_data.fk_hdrset[0];
 797         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 798         fm_data->fk_metadata |= FKM_GENEVE;
 799         fm_mask->fk_metadata |= FKM_GENEVE;
 800         /* items from here on out are inner header items, except options */
 801         arg->header_level = 1;
 802
 803         /* Match all if no spec */
 804         if (!spec)
 805                 return 0;
 806         if (!mask)
 807                 mask = &rte_flow_item_geneve_mask;
 808
 809         /*
 810          * Use the raw L4 buffer to match geneve as fm_header_set does
 811          * not have geneve header. A UDP item may precede the geneve
 812          * item. Using the raw buffer does not affect such UDP item,
 813          * since we skip UDP in the raw buffer.
 814          */
 815         fm_data->fk_header_select |= FKH_L4RAW;
 816         fm_mask->fk_header_select |= FKH_L4RAW;
 817         off = sizeof(fm_data->l4.udp);
 818         memcpy(&fm_data->l4.rawdata[off], spec, sizeof(struct rte_geneve_hdr));
 819         memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(struct rte_geneve_hdr));
 820         return 0;
 821 }
 822
 823 static int
 824 enic_fm_copy_item_geneve_opt(struct copy_item_args *arg)
 825 {
 826         const struct rte_flow_item *item = arg->item;
 827         const struct rte_flow_item_geneve_opt *spec = item->spec;
 828         const struct rte_flow_item_geneve_opt *mask = item->mask;
 829         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 830         struct fm_header_set *fm_data, *fm_mask;
 831         struct rte_geneve_hdr *geneve;
 832         int off, len;
 833
 834         ENICPMD_FUNC_TRACE();
 835         fm_data = &entry->ftm_data.fk_hdrset[0];
 836         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 837         /* Match all if no spec */
 838         if (!spec)
 839                 return 0;
 840         if (!mask)
 841                 mask = &rte_flow_item_geneve_opt_mask;
 842
 843         if (spec->option_len > 0 &&
 844             (spec->data == NULL || mask->data == NULL)) {
 845                 return rte_flow_error_set(arg->error, EINVAL,
 846                         RTE_FLOW_ERROR_TYPE_ITEM,
 847                         NULL, "enic: geneve_opt unexpected null data");
 848         }
 849         /*
 850          * Geneve item must already be in the raw buffer. Append the
 851          * option pattern to it. There are two limitations.
 852          * (1) Can match only the 1st option, the first one following Geneve
 853          * (2) Geneve header must specify option length, as HW does not
 854          *     have "has Geneve option" flag.
 855          */
 856         RTE_ASSERT((fm_data->fk_header_select & FKH_L4RAW) != 0);
 857         RTE_ASSERT((fm_mask->fk_header_select & FKH_L4RAW) != 0);
 858         off = sizeof(fm_data->l4.udp);
 859         geneve = (struct rte_geneve_hdr *)&fm_data->l4.rawdata[off];
 860         if (geneve->opt_len == 0) {
 861                 return rte_flow_error_set(arg->error, EINVAL,
 862                         RTE_FLOW_ERROR_TYPE_ITEM,
 863                         NULL, "enic: geneve_opt requires non-zero geneve option length");
 864         }
 865         geneve = (struct rte_geneve_hdr *)&fm_mask->l4.rawdata[off];
 866         if (geneve->opt_len == 0) {
 867                 return rte_flow_error_set(arg->error, EINVAL,
 868                         RTE_FLOW_ERROR_TYPE_ITEM,
 869                         NULL, "enic: geneve_opt requires non-zero geneve option length mask");
 870         }
 871         off = sizeof(fm_data->l4.udp) + sizeof(struct rte_geneve_hdr);
 872         if (off + (spec->option_len + 1) * 4 > FM_LAYER_SIZE) {
 873                 return rte_flow_error_set(arg->error, EINVAL,
 874                         RTE_FLOW_ERROR_TYPE_ITEM,
 875                         NULL, "enic: geneve_opt too large");
 876         }
 877         /* Copy option header */
 878         memcpy(&fm_data->l4.rawdata[off], spec, 4);
 879         memcpy(&fm_mask->l4.rawdata[off], mask, 4);
 880         /* Copy option data */
 881         if (spec->option_len > 0) {
 882                 off += 4;
 883                 len = spec->option_len * 4;
 884                 memcpy(&fm_data->l4.rawdata[off], spec->data, len);
 885                 memcpy(&fm_mask->l4.rawdata[off], mask->data, len);
 886         }
 887         return 0;
 888 }
 889
 890 /* Match eCPRI combined message header */
 891 static int
 892 enic_fm_copy_item_ecpri(struct copy_item_args *arg)
 893 {
 894         const struct rte_flow_item *item = arg->item;
 895         const struct rte_flow_item_ecpri *spec = item->spec;
 896         const struct rte_flow_item_ecpri *mask = item->mask;
 897         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 898         struct fm_header_set *fm_data, *fm_mask;
 899         uint8_t *fm_data_to, *fm_mask_to;
 900
 901         ENICPMD_FUNC_TRACE();
 902
 903         /* Tunneling not supported- only matching on inner eCPRI fields. */
 904         if (arg->header_level > 0)
 905                 return -EINVAL;
 906
 907         /* Need both spec and mask */
 908         if (!spec || !mask)
 909                 return -EINVAL;
 910
 911         fm_data = &entry->ftm_data.fk_hdrset[0];
 912         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 913
 914         /* eCPRI can only follow L2/VLAN layer if ethernet type is 0xAEFE. */
 915         if (!(fm_data->fk_metadata & FKM_UDP) &&
 916             (fm_mask->l2.eth.fk_ethtype != UINT16_MAX ||
 917             rte_cpu_to_be_16(fm_data->l2.eth.fk_ethtype) !=
 918             RTE_ETHER_TYPE_ECPRI))
 919                 return -EINVAL;
 920
 921         if (fm_data->fk_metadata & FKM_UDP) {
 922                 /* eCPRI on UDP */
 923                 fm_data->fk_header_select |= FKH_L4RAW;
 924                 fm_mask->fk_header_select |= FKH_L4RAW;
 925                 fm_data_to = &fm_data->l4.rawdata[sizeof(fm_data->l4.udp)];
 926                 fm_mask_to = &fm_mask->l4.rawdata[sizeof(fm_data->l4.udp)];
 927         } else {
 928                 /* eCPRI directly after Etherent header */
 929                 fm_data->fk_header_select |= FKH_L3RAW;
 930                 fm_mask->fk_header_select |= FKH_L3RAW;
 931                 fm_data_to = &fm_data->l3.rawdata[0];
 932                 fm_mask_to = &fm_mask->l3.rawdata[0];
 933         }
 934
 935         /*
 936          * Use the raw L3 or L4 buffer to match eCPRI since fm_header_set does
 937          * not have eCPRI header. Only 1st message header of PDU can be matched.
 938          * "C" * bit ignored.
 939          */
 940         memcpy(fm_data_to, spec, sizeof(*spec));
 941         memcpy(fm_mask_to, mask, sizeof(*mask));
 942         return 0;
 943 }
 944
 945 /*
 946  * Currently, raw pattern match is very limited. It is intended for matching
 947  * UDP tunnel header (e.g. vxlan or geneve).
 948  */
 949 static int
 950 enic_fm_copy_item_raw(struct copy_item_args *arg)
 951 {
 952         const struct rte_flow_item *item = arg->item;
 953         const struct rte_flow_item_raw *spec = item->spec;
 954         const struct rte_flow_item_raw *mask = item->mask;
 955         const uint8_t lvl = arg->header_level;
 956         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 957         struct fm_header_set *fm_data, *fm_mask;
 958
 959         ENICPMD_FUNC_TRACE();
 960         /* Cannot be used for inner packet */
 961         if (lvl > 0)
 962                 return -EINVAL;
 963         /* Need both spec and mask */
 964         if (!spec || !mask)
 965                 return -EINVAL;
 966         /* Only supports relative with offset 0 */
 967         if (!spec->relative || spec->offset != 0 || spec->search ||
 968             spec->limit)
 969                 return -EINVAL;
 970         /* Need non-null pattern that fits within the NIC's filter pattern */
 971         if (spec->length == 0 ||
 972             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 973             !spec->pattern || !mask->pattern)
 974                 return -EINVAL;
 975         /*
 976          * Mask fields, including length, are often set to zero. Assume that
 977          * means "same as spec" to avoid breaking existing apps. If length
 978          * is not zero, then it should be >= spec length.
 979          *
 980          * No more pattern follows this, so append to the L4 layer instead of
 981          * L5 to work with both recent and older VICs.
 982          */
 983         if (mask->length != 0 && mask->length < spec->length)
 984                 return -EINVAL;
 985
 986         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 987         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 988         fm_data->fk_header_select |= FKH_L4RAW;
 989         fm_mask->fk_header_select |= FKH_L4RAW;
 990         fm_data->fk_header_select &= ~FKH_UDP;
 991         fm_mask->fk_header_select &= ~FKH_UDP;
 992         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 993                spec->pattern, spec->length);
 994         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 995                mask->pattern, spec->length);
 996         return 0;
 997 }
 998
 999 static int
1000 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
1001 {
1002         return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
1003 }
1004
1005 static int
1006 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
1007                struct fm_key_template *key, int entries,
1008                struct enic_fm_fet **fet_out)
1009 {
1010         struct fm_exact_match_table *cmd;
1011         struct fm_header_set *hdr;
1012         struct enic_fm_fet *fet;
1013         uint64_t args[3];
1014         int ret;
1015
1016         ENICPMD_FUNC_TRACE();
1017         fet = calloc(1, sizeof(struct enic_fm_fet));
1018         if (fet == NULL)
1019                 return -ENOMEM;
1020         cmd = &fm->cmd.va->fm_exact_match_table;
1021         memset(cmd, 0, sizeof(*cmd));
1022         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
1023         cmd->fet_stage = FM_STAGE_LAST;
1024         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
1025         if (key == NULL) {
1026                 hdr = &cmd->fet_key.fk_hdrset[0];
1027                 memset(hdr, 0, sizeof(*hdr));
1028                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
1029                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
1030                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
1031                 hdr->l4.udp.fk_source = 0xFFFF;
1032                 hdr->l4.udp.fk_dest = 0xFFFF;
1033                 fet->default_key = 1;
1034         } else {
1035                 memcpy(&cmd->fet_key, key, sizeof(*key));
1036                 memcpy(&fet->key, key, sizeof(*key));
1037                 fet->default_key = 0;
1038         }
1039         cmd->fet_key.fk_packet_tag = 1;
1040
1041         args[0] = FM_EXACT_TABLE_ALLOC;
1042         args[1] = fm->cmd.pa;
1043         ret = flowman_cmd(fm, args, 2);
1044         if (ret) {
1045                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
1046                 free(fet);
1047                 return ret;
1048         }
1049         fet->handle = args[0];
1050         fet->ingress = ingress;
1051         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
1052                     fet->handle);
1053         *fet_out = fet;
1054         return 0;
1055 }
1056
1057 static void
1058 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
1059 {
1060         ENICPMD_FUNC_TRACE();
1061         enic_fm_tbl_free(fm, fet->handle);
1062         if (!fet->default_key)
1063                 TAILQ_REMOVE(&fm->fet_list, fet, list);
1064         free(fet);
1065 }
1066
1067 /*
1068  * Get the exact match table for the given combination of
1069  * <group, ingress, key>. Allocate one on the fly as necessary.
1070  */
1071 static int
1072 enic_fet_get(struct enic_flowman *fm,
1073              uint32_t group,
1074              uint8_t ingress,
1075              struct fm_key_template *key,
1076              struct enic_fm_fet **fet_out,
1077              struct rte_flow_error *error)
1078 {
1079         struct enic_fm_fet *fet;
1080
1081         ENICPMD_FUNC_TRACE();
1082         /* See if we already have this table open */
1083         TAILQ_FOREACH(fet, &fm->fet_list, list) {
1084                 if (fet->group == group && fet->ingress == ingress)
1085                         break;
1086         }
1087         if (fet == NULL) {
1088                 /* Jumping to a non-existing group? Use the default table */
1089                 if (key == NULL) {
1090                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
1091                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
1092                         return rte_flow_error_set(error, EINVAL,
1093                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1094                                 NULL, "enic: cannot get exact match table");
1095                 }
1096                 fet->group = group;
1097                 /* Default table is never on the open table list */
1098                 if (!fet->default_key)
1099                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
1100         }
1101         fet->ref++;
1102         *fet_out = fet;
1103         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
1104                     fet->default_key ? "default" : "",
1105                     fet->ingress ? "ingress" : "egress",
1106                     fet->group, fet->ref);
1107         return 0;
1108 }
1109
1110 static void
1111 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
1112 {
1113         ENICPMD_FUNC_TRACE();
1114         RTE_ASSERT(fet->ref > 0);
1115         fet->ref--;
1116         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
1117                     fet->default_key ? "default" : "",
1118                     fet->ingress ? "ingress" : "egress",
1119                     fet->group, fet->ref);
1120         if (fet->ref == 0)
1121                 enic_fet_free(fm, fet);
1122 }
1123
1124 /* Return 1 if current item is valid on top of the previous one. */
1125 static int
1126 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
1127                        const struct enic_fm_items *item_info,
1128                        uint8_t is_first_item)
1129 {
1130         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
1131
1132         ENICPMD_FUNC_TRACE();
1133         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
1134                 if (prev_item == *allowed_items)
1135                         return 1;
1136         }
1137
1138         /* This is the first item in the stack. Check if that's cool */
1139         if (is_first_item && item_info->valid_start_item)
1140                 return 1;
1141         return 0;
1142 }
1143
1144 /*
1145  * Build the flow manager match entry structure from the provided pattern.
1146  * The pattern is validated as the items are copied.
1147  */
1148 static int
1149 enic_fm_copy_entry(struct enic_flowman *fm,
1150                    const struct rte_flow_item pattern[],
1151                    struct rte_flow_error *error)
1152 {
1153         const struct enic_fm_items *item_info;
1154         enum rte_flow_item_type prev_item;
1155         const struct rte_flow_item *item;
1156         struct copy_item_args args;
1157         uint8_t prev_header_level;
1158         uint8_t is_first_item;
1159         int ret;
1160
1161         ENICPMD_FUNC_TRACE();
1162         item = pattern;
1163         is_first_item = 1;
1164         prev_item = RTE_FLOW_ITEM_TYPE_END;
1165
1166         args.fm_tcam_entry = &fm->tcam_entry;
1167         args.header_level = 0;
1168         prev_header_level = 0;
1169         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
1170                 /*
1171                  * Get info about how to validate and copy the item. If NULL
1172                  * is returned the nic does not support the item.
1173                  */
1174                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
1175                         continue;
1176
1177                 item_info = &enic_fm_items[item->type];
1178
1179                 if (item->type >= RTE_DIM(enic_fm_items) ||
1180                     item_info->copy_item == NULL) {
1181                         return rte_flow_error_set(error, ENOTSUP,
1182                                 RTE_FLOW_ERROR_TYPE_ITEM,
1183                                 NULL, "enic: unsupported item");
1184                 }
1185                 /*
1186                  * Check vNIC feature dependencies. Geneve item needs
1187                  * Geneve offload feature
1188                  */
1189                 if (item->type == RTE_FLOW_ITEM_TYPE_GENEVE &&
1190                     !fm->user_enic->geneve) {
1191                         return rte_flow_error_set(error, ENOTSUP,
1192                                 RTE_FLOW_ERROR_TYPE_ITEM,
1193                                 NULL, "enic: geneve not supported");
1194                 }
1195                 /* check to see if item stacking is valid */
1196                 if (!fm_item_stacking_valid(prev_item, item_info,
1197                                             is_first_item))
1198                         goto stacking_error;
1199
1200                 args.item = item;
1201                 args.error = error;
1202                 if (error)
1203                         error->type = RTE_FLOW_ERROR_TYPE_NONE;
1204                 ret = item_info->copy_item(&args);
1205                 if (ret) {
1206                         /* If copy_item set the error, return that */
1207                         if (error->type != RTE_FLOW_ERROR_TYPE_NONE)
1208                                 return ret;
1209                         goto item_not_supported;
1210                 }
1211                 /* Going from outer to inner? Treat it as a new packet start */
1212                 if (prev_header_level != args.header_level) {
1213                         prev_item = RTE_FLOW_ITEM_TYPE_END;
1214                         is_first_item = 1;
1215                 } else {
1216                         prev_item = item->type;
1217                         is_first_item = 0;
1218                 }
1219                 prev_header_level = args.header_level;
1220         }
1221         return 0;
1222
1223 item_not_supported:
1224         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
1225                                   NULL, "enic: unsupported item type");
1226
1227 stacking_error:
1228         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
1229                                   item, "enic: unsupported item stack");
1230 }
1231
1232 static void
1233 flow_item_skip_void(const struct rte_flow_item **item)
1234 {
1235         for ( ; ; (*item)++)
1236                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
1237                         return;
1238 }
1239
1240 static void
1241 append_template(void **template, uint8_t *off, const void *data, int len)
1242 {
1243         memcpy(*template, data, len);
1244         *template = (char *)*template + len;
1245         *off = *off + len;
1246 }
1247
1248 static int
1249 enic_fm_append_action_op(struct enic_flowman *fm,
1250                          struct fm_action_op *fm_op,
1251                          struct rte_flow_error *error)
1252 {
1253         int count;
1254
1255         count = fm->action_op_count;
1256         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
1257                     count, fm_op->fa_op);
1258         if (count == FM_ACTION_OP_MAX) {
1259                 return rte_flow_error_set(error, EINVAL,
1260                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1261                         "too many action operations");
1262         }
1263         fm->action.fma_action_ops[count] = *fm_op;
1264         fm->action_op_count = count + 1;
1265         return 0;
1266 }
1267
1268 static struct fm_action_op *
1269 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
1270 {
1271         struct fm_action_op *op;
1272         int i;
1273
1274         for (i = 0; i < fm->action_op_count; i++) {
1275                 op = &fm->action.fma_action_ops[i];
1276                 if (op->fa_op == opcode)
1277                         return op;
1278         }
1279         return NULL;
1280 }
1281
1282 /* NIC requires that 1st steer appear before decap.
1283  * Correct example: steer, decap, steer, steer, ...
1284  */
1285 static void
1286 enic_fm_reorder_action_op(struct enic_flowman *fm)
1287 {
1288         struct fm_action_op *op, *steer, *decap;
1289         struct fm_action_op tmp_op;
1290
1291         ENICPMD_FUNC_TRACE();
1292         /* Find 1st steer and decap */
1293         op = fm->action.fma_action_ops;
1294         steer = NULL;
1295         decap = NULL;
1296         while (op->fa_op != FMOP_END) {
1297                 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
1298                                op->fa_op == FMOP_DECAP_STRIP))
1299                         decap = op;
1300                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
1301                         steer = op;
1302                 op++;
1303         }
1304         /* If decap is before steer, swap */
1305         if (steer && decap && decap < steer) {
1306                 op = fm->action.fma_action_ops;
1307                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
1308                             (long)(decap - op), (long)(steer - op));
1309                 tmp_op = *decap;
1310                 *decap = *steer;
1311                 *steer = tmp_op;
1312         }
1313 }
1314
1315 /* VXLAN decap is done via flowman compound action */
1316 static int
1317 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
1318                          struct fm_tcam_match_entry *fmt,
1319                          const struct rte_flow_action *action,
1320                          struct rte_flow_error *error)
1321 {
1322         struct fm_header_set *fm_data;
1323         struct fm_action_op fm_op;
1324
1325         ENICPMD_FUNC_TRACE();
1326         fm_data = &fmt->ftm_data.fk_hdrset[0];
1327         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
1328                 return rte_flow_error_set(error, EINVAL,
1329                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1330                         "vxlan-decap: vxlan must be in pattern");
1331         }
1332
1333         memset(&fm_op, 0, sizeof(fm_op));
1334         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
1335         return enic_fm_append_action_op(fm, &fm_op, error);
1336 }
1337
1338 /* Generate a reasonable source port number */
1339 static uint16_t
1340 gen_src_port(void)
1341 {
1342         /* Min/max below are the default values in OVS-DPDK and Linux */
1343         uint16_t p = rte_rand();
1344         p = RTE_MAX(p, 32768);
1345         p = RTE_MIN(p, 61000);
1346         return rte_cpu_to_be_16(p);
1347 }
1348
1349 /* VXLAN encap is done via flowman compound action */
1350 static int
1351 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1352                          const struct rte_flow_item *item,
1353                          struct rte_flow_error *error)
1354 {
1355         struct fm_action_op fm_op;
1356         struct rte_ether_hdr *eth;
1357         struct rte_udp_hdr *udp;
1358         uint16_t *ethertype;
1359         void *template;
1360         uint8_t off;
1361
1362         ENICPMD_FUNC_TRACE();
1363         memset(&fm_op, 0, sizeof(fm_op));
1364         fm_op.fa_op = FMOP_ENCAP;
1365         template = fm->action.fma_data;
1366         off = 0;
1367         /*
1368          * Copy flow items to the flowman template starting L2.
1369          * L2 must be ethernet.
1370          */
1371         flow_item_skip_void(&item);
1372         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1373                 return rte_flow_error_set(error, EINVAL,
1374                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1375                         "vxlan-encap: first item should be ethernet");
1376         eth = (struct rte_ether_hdr *)template;
1377         ethertype = &eth->ether_type;
1378         append_template(&template, &off, item->spec,
1379                         sizeof(struct rte_ether_hdr));
1380         item++;
1381         flow_item_skip_void(&item);
1382         /* Optional VLAN */
1383         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1384                 const struct rte_flow_item_vlan *spec;
1385
1386                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1387                 spec = item->spec;
1388                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1389                 item++;
1390                 flow_item_skip_void(&item);
1391         }
1392         /* L3 must be IPv4, IPv6 */
1393         switch (item->type) {
1394         case RTE_FLOW_ITEM_TYPE_IPV4:
1395         {
1396                 struct rte_ipv4_hdr *ip4;
1397
1398                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1399                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1400                 ip4 = (struct rte_ipv4_hdr *)template;
1401                 /*
1402                  * Offset of IPv4 length field and its initial value
1403                  * (IP + UDP + VXLAN) are specified in the action. The NIC
1404                  * will add inner packet length.
1405                  */
1406                 fm_op.encap.len1_offset = off +
1407                         offsetof(struct rte_ipv4_hdr, total_length);
1408                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1409                         sizeof(struct rte_udp_hdr) +
1410                         sizeof(struct rte_vxlan_hdr);
1411                 append_template(&template, &off, item->spec,
1412                                 sizeof(struct rte_ipv4_hdr));
1413                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1414                 if (ip4->time_to_live == 0)
1415                         ip4->time_to_live = IP_DEFTTL;
1416                 ip4->next_proto_id = IPPROTO_UDP;
1417                 break;
1418         }
1419         case RTE_FLOW_ITEM_TYPE_IPV6:
1420         {
1421                 struct rte_ipv6_hdr *ip6;
1422
1423                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1424                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1425                 ip6 = (struct rte_ipv6_hdr *)template;
1426                 fm_op.encap.len1_offset = off +
1427                         offsetof(struct rte_ipv6_hdr, payload_len);
1428                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1429                         sizeof(struct rte_vxlan_hdr);
1430                 append_template(&template, &off, item->spec,
1431                                 sizeof(struct rte_ipv6_hdr));
1432                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1433                 if (ip6->hop_limits == 0)
1434                         ip6->hop_limits = IP_DEFTTL;
1435                 ip6->proto = IPPROTO_UDP;
1436                 break;
1437         }
1438         default:
1439                 return rte_flow_error_set(error,
1440                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1441                         "vxlan-encap: L3 must be IPv4/IPv6");
1442         }
1443         item++;
1444         flow_item_skip_void(&item);
1445
1446         /* L4 is UDP */
1447         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1448                 return rte_flow_error_set(error, EINVAL,
1449                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1450                         "vxlan-encap: UDP must follow IPv4/IPv6");
1451         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1452         fm_op.encap.len2_offset =
1453                 off + offsetof(struct rte_udp_hdr, dgram_len);
1454         fm_op.encap.len2_delta =
1455                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1456         udp = (struct rte_udp_hdr *)template;
1457         append_template(&template, &off, item->spec,
1458                         sizeof(struct rte_udp_hdr));
1459         /*
1460          * Firmware does not hash/fill source port yet. Generate a
1461          * random port, as there is *usually* one rte_flow for the
1462          * given inner packet stream (i.e. a single stream has one
1463          * random port).
1464          */
1465         if (udp->src_port == 0)
1466                 udp->src_port = gen_src_port();
1467         item++;
1468         flow_item_skip_void(&item);
1469
1470         /* Finally VXLAN */
1471         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1472                 return rte_flow_error_set(error,
1473                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1474                         "vxlan-encap: VXLAN must follow UDP");
1475         append_template(&template, &off, item->spec,
1476                         sizeof(struct rte_flow_item_vxlan));
1477
1478         /*
1479          * Fill in the rest of the action structure.
1480          * Indicate that we want to encap with vxlan at packet start.
1481          */
1482         fm_op.encap.template_offset = 0;
1483         fm_op.encap.template_len = off;
1484         return enic_fm_append_action_op(fm, &fm_op, error);
1485 }
1486
1487 static int
1488 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1489                   uint64_t *handle)
1490 {
1491         uint32_t bdf;
1492         uint64_t args[2];
1493         int rc;
1494
1495         ENICPMD_FUNC_TRACE();
1496         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1497                     addr->function);
1498         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1499         args[0] = FM_VNIC_FIND;
1500         args[1] = bdf;
1501         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1502         if (rc != 0) {
1503                 /* Expected to fail if BDF is not on the adapter */
1504                 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1505                 return rc;
1506         }
1507         *handle = args[0];
1508         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1509         return 0;
1510 }
1511
1512 /*
1513  * Egress: target port should be either PF uplink or VF.
1514  * Supported cases
1515  * 1. VF egress -> PF uplink
1516  *   PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1517  * 2. VF egress -> VF
1518  *
1519  * Unsupported cases
1520  * 1. PF egress -> VF
1521  *   App should be using representor to pass packets to VF
1522  */
1523 static int
1524 vf_egress_port_id_action(struct enic_flowman *fm,
1525                          struct rte_eth_dev *dst_dev,
1526                          uint64_t dst_vnic_h,
1527                          struct fm_action_op *fm_op,
1528                          struct rte_flow_error *error)
1529 {
1530         struct enic *src_enic, *dst_enic;
1531         struct enic_vf_representor *vf;
1532         uint8_t uif;
1533         int ret;
1534
1535         ENICPMD_FUNC_TRACE();
1536         src_enic = fm->user_enic;
1537         dst_enic = pmd_priv(dst_dev);
1538         if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1539                 return rte_flow_error_set(error, EINVAL,
1540                         RTE_FLOW_ERROR_TYPE_ACTION,
1541                         NULL, "source port is not VF representor");
1542         }
1543
1544         /* VF -> PF uplink. dst is not VF representor */
1545         if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1546                 /* PF is the VF's PF? Then nothing to do */
1547                 vf = VF_ENIC_TO_VF_REP(src_enic);
1548                 if (vf->pf == dst_enic) {
1549                         ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1550                         return 0;
1551                 }
1552                 /* If not, steer to the remote PF's uplink */
1553                 uif = dst_enic->fm_vnic_uif;
1554                 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1555                 memset(fm_op, 0, sizeof(*fm_op));
1556                 fm_op->fa_op = FMOP_SET_EGPORT;
1557                 fm_op->set_egport.egport = uif;
1558                 ret = enic_fm_append_action_op(fm, fm_op, error);
1559                 return ret;
1560         }
1561
1562         /* VF -> VF loopback. Hairpin and steer to vnic */
1563         memset(fm_op, 0, sizeof(*fm_op));
1564         fm_op->fa_op = FMOP_EG_HAIRPIN;
1565         ret = enic_fm_append_action_op(fm, fm_op, error);
1566         if (ret)
1567                 return ret;
1568         ENICPMD_LOG(DEBUG, "egress hairpin");
1569         fm->hairpin_steer_vnic_h = dst_vnic_h;
1570         fm->need_hairpin_steer = 1;
1571         return 0;
1572 }
1573
1574 static int
1575 enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
1576                            struct rte_eth_dev **dst_dev,
1577                            struct rte_flow_error *error)
1578 {
1579         struct rte_eth_dev *dev;
1580
1581         ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
1582         if (!rte_eth_dev_is_valid_port(dst_port_id)) {
1583                 return rte_flow_error_set(error, EINVAL,
1584                         RTE_FLOW_ERROR_TYPE_ACTION,
1585                         NULL, "invalid port_id");
1586         }
1587         dev = &rte_eth_devices[dst_port_id];
1588         if (!dev_is_enic(dev)) {
1589                 return rte_flow_error_set(error, EINVAL,
1590                         RTE_FLOW_ERROR_TYPE_ACTION,
1591                         NULL, "port_id is not enic");
1592         }
1593         if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
1594                 return rte_flow_error_set(error, EINVAL,
1595                         RTE_FLOW_ERROR_TYPE_ACTION,
1596                         NULL, "destination and source ports are not in the same switch domain");
1597         }
1598
1599         *dst_dev = dev;
1600         return 0;
1601 }
1602
1603 /* Translate flow actions to flowman TCAM entry actions */
1604 static int
1605 enic_fm_copy_action(struct enic_flowman *fm,
1606                     const struct rte_flow_action actions[],
1607                     uint8_t ingress,
1608                     struct rte_flow_error *error)
1609 {
1610         enum {
1611                 FATE = 1 << 0,
1612                 DECAP = 1 << 1,
1613                 PASSTHRU = 1 << 2,
1614                 COUNT = 1 << 3,
1615                 ENCAP = 1 << 4,
1616                 PUSH_VLAN = 1 << 5,
1617                 PORT_ID = 1 << 6,
1618         };
1619         struct fm_tcam_match_entry *fmt;
1620         struct fm_action_op fm_op;
1621         bool need_ovlan_action;
1622         struct enic *enic;
1623         uint32_t overlap;
1624         uint64_t vnic_h;
1625         uint16_t ovlan;
1626         bool first_rq;
1627         bool steer;
1628         int ret;
1629
1630         ENICPMD_FUNC_TRACE();
1631         fmt = &fm->tcam_entry;
1632         need_ovlan_action = false;
1633         ovlan = 0;
1634         first_rq = true;
1635         steer = false;
1636         enic = fm->user_enic;
1637         overlap = 0;
1638         vnic_h = enic->fm_vnic_handle;
1639
1640         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1641                 switch (actions->type) {
1642                 case RTE_FLOW_ACTION_TYPE_VOID:
1643                         continue;
1644                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1645                         if (overlap & PASSTHRU)
1646                                 goto unsupported;
1647                         overlap |= PASSTHRU;
1648                         break;
1649                 }
1650                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1651                         const struct rte_flow_action_jump *jump =
1652                                 actions->conf;
1653                         struct enic_fm_fet *fet;
1654
1655                         if (overlap & FATE)
1656                                 goto unsupported;
1657                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1658                                            &fet, error);
1659                         if (ret)
1660                                 return ret;
1661                         overlap |= FATE;
1662                         memset(&fm_op, 0, sizeof(fm_op));
1663                         fm_op.fa_op = FMOP_EXACT_MATCH;
1664                         fm_op.exact.handle = fet->handle;
1665                         fm->fet = fet;
1666                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1667                         if (ret)
1668                                 return ret;
1669                         break;
1670                 }
1671                 case RTE_FLOW_ACTION_TYPE_MARK: {
1672                         const struct rte_flow_action_mark *mark =
1673                                 actions->conf;
1674
1675                         if (enic->use_noscatter_vec_rx_handler)
1676                                 goto unsupported;
1677                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1678                                 return rte_flow_error_set(error, EINVAL,
1679                                         RTE_FLOW_ERROR_TYPE_ACTION,
1680                                         NULL, "invalid mark id");
1681                         memset(&fm_op, 0, sizeof(fm_op));
1682                         fm_op.fa_op = FMOP_MARK;
1683                         fm_op.mark.mark = mark->id + 1;
1684                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1685                         if (ret)
1686                                 return ret;
1687                         break;
1688                 }
1689                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1690                         if (enic->use_noscatter_vec_rx_handler)
1691                                 goto unsupported;
1692                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1693                         memset(&fm_op, 0, sizeof(fm_op));
1694                         fm_op.fa_op = FMOP_MARK;
1695                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1696                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1697                         if (ret)
1698                                 return ret;
1699                         break;
1700                 }
1701                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1702                         const struct rte_flow_action_queue *queue =
1703                                 actions->conf;
1704
1705                         /*
1706                          * If fate other than QUEUE or RSS, fail. Multiple
1707                          * rss and queue actions are ok.
1708                          */
1709                         if ((overlap & FATE) && first_rq)
1710                                 goto unsupported;
1711                         first_rq = false;
1712                         overlap |= FATE;
1713                         memset(&fm_op, 0, sizeof(fm_op));
1714                         fm_op.fa_op = FMOP_RQ_STEER;
1715                         fm_op.rq_steer.rq_index =
1716                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1717                         fm_op.rq_steer.rq_count = 1;
1718                         fm_op.rq_steer.vnic_handle = vnic_h;
1719                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1720                         if (ret)
1721                                 return ret;
1722                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1723                                     fm_op.rq_steer.rq_index);
1724                         steer = true;
1725                         break;
1726                 }
1727                 case RTE_FLOW_ACTION_TYPE_DROP: {
1728                         if (overlap & FATE)
1729                                 goto unsupported;
1730                         overlap |= FATE;
1731                         memset(&fm_op, 0, sizeof(fm_op));
1732                         fm_op.fa_op = FMOP_DROP;
1733                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1734                         if (ret)
1735                                 return ret;
1736                         ENICPMD_LOG(DEBUG, "create DROP action");
1737                         break;
1738                 }
1739                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1740                         if (overlap & COUNT)
1741                                 goto unsupported;
1742                         overlap |= COUNT;
1743                         /* Count is associated with entry not action on VIC. */
1744                         fmt->ftm_flags |= FMEF_COUNTER;
1745                         break;
1746                 }
1747                 case RTE_FLOW_ACTION_TYPE_RSS: {
1748                         const struct rte_flow_action_rss *rss = actions->conf;
1749                         bool allow;
1750                         uint16_t i;
1751
1752                         /*
1753                          * If fate other than QUEUE or RSS, fail. Multiple
1754                          * rss and queue actions are ok.
1755                          */
1756                         if ((overlap & FATE) && first_rq)
1757                                 goto unsupported;
1758                         first_rq = false;
1759                         overlap |= FATE;
1760
1761                         /*
1762                          * Hardware only supports RSS actions on outer level
1763                          * with default type and function. Queues must be
1764                          * sequential.
1765                          */
1766                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1767                                 rss->level == 0 && (rss->types == 0 ||
1768                                 rss->types == enic->rss_hf) &&
1769                                 rss->queue_num <= enic->rq_count &&
1770                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1771
1772
1773                         /* Identity queue map needs to be sequential */
1774                         for (i = 1; i < rss->queue_num; i++)
1775                                 allow = allow && (rss->queue[i] ==
1776                                         rss->queue[i - 1] + 1);
1777                         if (!allow)
1778                                 goto unsupported;
1779
1780                         memset(&fm_op, 0, sizeof(fm_op));
1781                         fm_op.fa_op = FMOP_RQ_STEER;
1782                         fm_op.rq_steer.rq_index =
1783                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1784                         fm_op.rq_steer.rq_count = rss->queue_num;
1785                         fm_op.rq_steer.vnic_handle = vnic_h;
1786                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1787                         if (ret)
1788                                 return ret;
1789                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1790                                     fm_op.rq_steer.rq_index);
1791                         steer = true;
1792                         break;
1793                 }
1794                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1795                         const struct rte_flow_action_port_id *port;
1796                         struct rte_eth_dev *dev = NULL;
1797
1798                         if (!ingress && (overlap & PORT_ID)) {
1799                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1800                                 goto unsupported;
1801                         }
1802                         port = actions->conf;
1803                         if (port->original) {
1804                                 vnic_h = enic->fm_vnic_handle; /* This port */
1805                                 break;
1806                         }
1807                         ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
1808                                                          error);
1809                         if (ret)
1810                                 return ret;
1811                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1812                         overlap |= PORT_ID;
1813                         /*
1814                          * Ingress. Nothing more to do. We add an implicit
1815                          * steer at the end if needed.
1816                          */
1817                         if (ingress)
1818                                 break;
1819                         /* Egress */
1820                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1821                                 error);
1822                         if (ret)
1823                                 return ret;
1824                         break;
1825                 }
1826                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1827                         if (overlap & DECAP)
1828                                 goto unsupported;
1829                         overlap |= DECAP;
1830
1831                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1832                                 error);
1833                         if (ret != 0)
1834                                 return ret;
1835                         break;
1836                 }
1837                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1838                         const struct rte_flow_action_vxlan_encap *encap;
1839
1840                         encap = actions->conf;
1841                         if (overlap & ENCAP)
1842                                 goto unsupported;
1843                         overlap |= ENCAP;
1844                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1845                                 error);
1846                         if (ret != 0)
1847                                 return ret;
1848                         break;
1849                 }
1850                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1851                         struct fm_action_op *decap;
1852
1853                         /*
1854                          * If decap-nostrip appears before pop vlan, this pop
1855                          * applies to the inner packet vlan. Turn it into
1856                          * decap-strip.
1857                          */
1858                         decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1859                         if (decap) {
1860                                 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1861                                 decap->fa_op = FMOP_DECAP_STRIP;
1862                                 break;
1863                         }
1864                         memset(&fm_op, 0, sizeof(fm_op));
1865                         fm_op.fa_op = FMOP_POP_VLAN;
1866                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1867                         if (ret)
1868                                 return ret;
1869                         break;
1870                 }
1871                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1872                         const struct rte_flow_action_of_push_vlan *vlan;
1873
1874                         if (overlap & PASSTHRU)
1875                                 goto unsupported;
1876                         vlan = actions->conf;
1877                         if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1878                                 return rte_flow_error_set(error, EINVAL,
1879                                         RTE_FLOW_ERROR_TYPE_ACTION,
1880                                         NULL, "unexpected push_vlan ethertype");
1881                         }
1882                         overlap |= PUSH_VLAN;
1883                         need_ovlan_action = true;
1884                         break;
1885                 }
1886                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1887                         const struct rte_flow_action_of_set_vlan_pcp *pcp;
1888
1889                         pcp = actions->conf;
1890                         if (pcp->vlan_pcp > 7) {
1891                                 return rte_flow_error_set(error, EINVAL,
1892                                         RTE_FLOW_ERROR_TYPE_ACTION,
1893                                         NULL, "invalid vlan_pcp");
1894                         }
1895                         need_ovlan_action = true;
1896                         ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1897                         break;
1898                 }
1899                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1900                         const struct rte_flow_action_of_set_vlan_vid *vid;
1901
1902                         vid = actions->conf;
1903                         need_ovlan_action = true;
1904                         ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1905                         break;
1906                 }
1907                 case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
1908                         const struct rte_flow_action_ethdev *ethdev;
1909                         struct rte_eth_dev *dev = NULL;
1910
1911                         ethdev = actions->conf;
1912                         ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1913                                                          &dev, error);
1914                         if (ret)
1915                                 return ret;
1916                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1917                         overlap |= PORT_ID;
1918                         /*
1919                          * Action PORT_REPRESENTOR implies ingress destination.
1920                          * Noting to do. We add an implicit stree at the
1921                          * end if needed.
1922                          */
1923                         ingress = 1;
1924                         break;
1925                 }
1926                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
1927                         const struct rte_flow_action_ethdev *ethdev;
1928                         struct rte_eth_dev *dev = NULL;
1929
1930                         if (overlap & PORT_ID) {
1931                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1932                                 goto unsupported;
1933                         }
1934                         ethdev = actions->conf;
1935                         ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1936                                                          &dev, error);
1937                         if (ret)
1938                                 return ret;
1939                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1940                         overlap |= PORT_ID;
1941                         /* Action REPRESENTED_PORT: always egress destination */
1942                         ingress = 0;
1943                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1944                                 error);
1945                         if (ret)
1946                                 return ret;
1947                         break;
1948                 }
1949                 default:
1950                         goto unsupported;
1951                 }
1952         }
1953
1954         if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1955                 goto unsupported;
1956         /* Egress from VF: need implicit WQ match */
1957         if (enic_is_vf_rep(enic) && !ingress) {
1958                 fmt->ftm_data.fk_wq_id = 0;
1959                 fmt->ftm_mask.fk_wq_id = 0xffff;
1960                 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1961                 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1962                             VF_ENIC_TO_VF_REP(enic)->vf_id);
1963         }
1964         if (need_ovlan_action) {
1965                 memset(&fm_op, 0, sizeof(fm_op));
1966                 fm_op.fa_op = FMOP_SET_OVLAN;
1967                 fm_op.ovlan.vlan = ovlan;
1968                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1969                 if (ret)
1970                         return ret;
1971         }
1972         /* Add steer op for PORT_ID without QUEUE */
1973         if ((overlap & PORT_ID) && !steer && ingress) {
1974                 memset(&fm_op, 0, sizeof(fm_op));
1975                 /* Always to queue 0 for now as generic RSS is not available */
1976                 fm_op.fa_op = FMOP_RQ_STEER;
1977                 fm_op.rq_steer.rq_index = 0;
1978                 fm_op.rq_steer.vnic_handle = vnic_h;
1979                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1980                 if (ret)
1981                         return ret;
1982                 ENICPMD_LOG(DEBUG, "add implicit steer op");
1983         }
1984         /* Add required END */
1985         memset(&fm_op, 0, sizeof(fm_op));
1986         fm_op.fa_op = FMOP_END;
1987         ret = enic_fm_append_action_op(fm, &fm_op, error);
1988         if (ret)
1989                 return ret;
1990         enic_fm_reorder_action_op(fm);
1991         return 0;
1992
1993 unsupported:
1994         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1995                                   NULL, "enic: unsupported action");
1996 }
1997
1998 /** Check if the action is supported */
1999 static int
2000 enic_fm_match_action(const struct rte_flow_action *action,
2001                      const enum rte_flow_action_type *supported_actions)
2002 {
2003         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
2004              supported_actions++) {
2005                 if (action->type == *supported_actions)
2006                         return 1;
2007         }
2008         return 0;
2009 }
2010
2011 /* Debug function to dump internal NIC action structure. */
2012 static void
2013 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
2014 {
2015         /* Manually keep in sync with FMOP commands */
2016         const char *fmop_str[FMOP_OP_MAX] = {
2017                 [FMOP_END] = "end",
2018                 [FMOP_DROP] = "drop",
2019                 [FMOP_RQ_STEER] = "steer",
2020                 [FMOP_EXACT_MATCH] = "exmatch",
2021                 [FMOP_MARK] = "mark",
2022                 [FMOP_EXT_MARK] = "ext_mark",
2023                 [FMOP_TAG] = "tag",
2024                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
2025                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
2026                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
2027                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
2028                 [FMOP_ENCAP] = "encap",
2029                 [FMOP_SET_OVLAN] = "set_ovlan",
2030                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
2031                 [FMOP_DECAP_STRIP] = "decap_strip",
2032                 [FMOP_POP_VLAN] = "pop_vlan",
2033                 [FMOP_SET_EGPORT] = "set_egport",
2034                 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
2035                 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
2036                 [FMOP_EMIT] = "emit",
2037                 [FMOP_MODIFY] = "modify",
2038         };
2039         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
2040         char buf[128], *bp = buf;
2041         const char *op_str;
2042         int i, n, buf_len;
2043
2044         buf[0] = '\0';
2045         buf_len = sizeof(buf);
2046         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
2047                 if (op->fa_op == FMOP_END)
2048                         break;
2049                 if (op->fa_op >= FMOP_OP_MAX)
2050                         op_str = "unknown";
2051                 else
2052                         op_str = fmop_str[op->fa_op];
2053                 n = snprintf(bp, buf_len, "%s,", op_str);
2054                 if (n > 0 && n < buf_len) {
2055                         bp += n;
2056                         buf_len -= n;
2057                 }
2058                 op++;
2059         }
2060         /* Remove trailing comma */
2061         if (buf[0])
2062                 *(bp - 1) = '\0';
2063         ENICPMD_LOG(DEBUG, "       Actions: %s", buf);
2064 }
2065
2066 static int
2067 bits_to_str(uint32_t bits, const char *strings[], int max,
2068             char *buf, int buf_len)
2069 {
2070         int i, n = 0, len = 0;
2071
2072         for (i = 0; i < max; i++) {
2073                 if (bits & (1 << i)) {
2074                         n = snprintf(buf, buf_len, "%s,", strings[i]);
2075                         if (n > 0 && n < buf_len) {
2076                                 buf += n;
2077                                 buf_len -= n;
2078                                 len += n;
2079                         }
2080                 }
2081         }
2082         /* Remove trailing comma */
2083         if (len) {
2084                 *(buf - 1) = '\0';
2085                 len--;
2086         }
2087         return len;
2088 }
2089
2090 /* Debug function to dump internal NIC filter structure. */
2091 static void
2092 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
2093                           int buf_len)
2094 {
2095         /* Manually keep in sync with FKM_BITS */
2096         const char *fm_fkm_str[FKM_BIT_COUNT] = {
2097                 [FKM_QTAG_BIT] = "qtag",
2098                 [FKM_CMD_BIT] = "cmd",
2099                 [FKM_IPV4_BIT] = "ip4",
2100                 [FKM_IPV6_BIT] = "ip6",
2101                 [FKM_ROCE_BIT] = "roce",
2102                 [FKM_UDP_BIT] = "udp",
2103                 [FKM_TCP_BIT] = "tcp",
2104                 [FKM_TCPORUDP_BIT] = "tcpportudp",
2105                 [FKM_IPFRAG_BIT] = "ipfrag",
2106                 [FKM_NVGRE_BIT] = "nvgre",
2107                 [FKM_VXLAN_BIT] = "vxlan",
2108                 [FKM_GENEVE_BIT] = "geneve",
2109                 [FKM_NSH_BIT] = "nsh",
2110                 [FKM_ROCEV2_BIT] = "rocev2",
2111                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
2112                 [FKM_IPOK_BIT] = "ipok",
2113                 [FKM_L4OK_BIT] = "l4ok",
2114                 [FKM_ROCEOK_BIT] = "roceok",
2115                 [FKM_FCSOK_BIT] = "fcsok",
2116                 [FKM_EG_SPAN_BIT] = "eg_span",
2117                 [FKM_IG_SPAN_BIT] = "ig_span",
2118                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
2119         };
2120         /* Manually keep in sync with FKH_BITS */
2121         const char *fm_fkh_str[FKH_BIT_COUNT] = {
2122                 [FKH_ETHER_BIT] = "eth",
2123                 [FKH_QTAG_BIT] = "qtag",
2124                 [FKH_L2RAW_BIT] = "l2raw",
2125                 [FKH_IPV4_BIT] = "ip4",
2126                 [FKH_IPV6_BIT] = "ip6",
2127                 [FKH_L3RAW_BIT] = "l3raw",
2128                 [FKH_UDP_BIT] = "udp",
2129                 [FKH_TCP_BIT] = "tcp",
2130                 [FKH_ICMP_BIT] = "icmp",
2131                 [FKH_VXLAN_BIT] = "vxlan",
2132                 [FKH_L4RAW_BIT] = "l4raw",
2133         };
2134         uint32_t fkh_bits = fk_hdrset->fk_header_select;
2135         uint32_t fkm_bits = fk_hdrset->fk_metadata;
2136         int n;
2137
2138         if (!fkm_bits && !fkh_bits)
2139                 return;
2140         n = snprintf(buf, buf_len, "metadata(");
2141         if (n > 0 && n < buf_len) {
2142                 buf += n;
2143                 buf_len -= n;
2144         }
2145         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
2146         if (n > 0 && n < buf_len) {
2147                 buf += n;
2148                 buf_len -= n;
2149         }
2150         n = snprintf(buf, buf_len, ") valid hdr fields(");
2151         if (n > 0 && n < buf_len) {
2152                 buf += n;
2153                 buf_len -= n;
2154         }
2155         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
2156         if (n > 0 && n < buf_len) {
2157                 buf += n;
2158                 buf_len -= n;
2159         }
2160         snprintf(buf, buf_len, ")");
2161 }
2162
2163 static void
2164 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
2165                         uint8_t ingress)
2166 {
2167         char buf[256];
2168
2169         memset(buf, 0, sizeof(buf));
2170         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
2171                                   buf, sizeof(buf));
2172         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
2173                     (ingress) ? "IG" : "EG", buf,
2174                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
2175                     match->ftm_position);
2176         memset(buf, 0, sizeof(buf));
2177         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
2178                                   buf, sizeof(buf));
2179         if (buf[0])
2180                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
2181 }
2182
2183 /* Debug function to dump internal NIC flow structures. */
2184 static void
2185 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
2186                         const struct fm_action *fm_action,
2187                         uint8_t ingress)
2188 {
2189         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
2190                 return;
2191         enic_fm_dump_tcam_match(fm_match, ingress);
2192         enic_fm_dump_tcam_actions(fm_action);
2193 }
2194
2195 static int
2196 enic_fm_flow_parse(struct enic_flowman *fm,
2197                    const struct rte_flow_attr *attrs,
2198                    const struct rte_flow_item pattern[],
2199                    const struct rte_flow_action actions[],
2200                    struct rte_flow_error *error)
2201 {
2202         const struct rte_flow_action *action;
2203         unsigned int ret;
2204         static const enum rte_flow_action_type *sa;
2205
2206         ENICPMD_FUNC_TRACE();
2207         ret = 0;
2208         if (!pattern) {
2209                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
2210                                    NULL, "no pattern specified");
2211                 return -rte_errno;
2212         }
2213
2214         if (!actions) {
2215                 rte_flow_error_set(error, EINVAL,
2216                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
2217                                    NULL, "no action specified");
2218                 return -rte_errno;
2219         }
2220
2221         if (attrs) {
2222                 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
2223                         rte_flow_error_set(error, ENOTSUP,
2224                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2225                                            NULL,
2226                                            "priorities are not supported for non-default (0) groups");
2227                         return -rte_errno;
2228                 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
2229                         rte_flow_error_set(error, ENOTSUP,
2230                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2231                                            NULL,
2232                                            "transfer is not supported");
2233                         return -rte_errno;
2234                 } else if (attrs->ingress && attrs->egress) {
2235                         rte_flow_error_set(error, ENOTSUP,
2236                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2237                                            NULL,
2238                                            "bidirectional rules not supported");
2239                         return -rte_errno;
2240                 }
2241
2242         } else {
2243                 rte_flow_error_set(error, EINVAL,
2244                                    RTE_FLOW_ERROR_TYPE_ATTR,
2245                                    NULL, "no attribute specified");
2246                 return -rte_errno;
2247         }
2248
2249         /* Verify Actions. */
2250         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
2251              enic_fm_supported_eg_actions;
2252         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
2253              action++) {
2254                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
2255                         continue;
2256                 else if (!enic_fm_match_action(action, sa))
2257                         break;
2258         }
2259         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
2260                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
2261                                    action, "invalid action");
2262                 return -rte_errno;
2263         }
2264         ret = enic_fm_copy_entry(fm, pattern, error);
2265         if (ret)
2266                 return ret;
2267         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
2268         return ret;
2269 }
2270
2271 static void
2272 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2273 {
2274         if (!fm_flow->counter_valid)
2275                 return;
2276         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
2277         fm_flow->counter_valid = false;
2278 }
2279
2280 static int
2281 enic_fm_more_counters(struct enic_flowman *fm)
2282 {
2283         struct enic_fm_counter *new_stack;
2284         struct enic_fm_counter *ctrs;
2285         int i, rc;
2286         uint64_t args[2];
2287
2288         ENICPMD_FUNC_TRACE();
2289         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
2290                                 FM_COUNTERS_EXPAND) *
2291                                 sizeof(struct enic_fm_counter), 0);
2292         if (new_stack == NULL) {
2293                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
2294                 return -ENOMEM;
2295         }
2296         fm->counter_stack = new_stack;
2297
2298         args[0] = FM_COUNTER_BRK;
2299         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
2300         rc = flowman_cmd(fm, args, 2);
2301         if (rc != 0) {
2302                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
2303                 return rc;
2304         }
2305         ctrs = (struct enic_fm_counter *)fm->counter_stack +
2306                 fm->counters_alloced;
2307         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
2308                 ctrs->handle = fm->counters_alloced + i;
2309                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
2310         }
2311         fm->counters_alloced += FM_COUNTERS_EXPAND;
2312         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
2313                     FM_COUNTERS_EXPAND, fm->counters_alloced);
2314         return 0;
2315 }
2316
2317 static int
2318 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
2319 {
2320         uint64_t args[3];
2321         int ret;
2322
2323         ENICPMD_FUNC_TRACE();
2324         args[0] = FM_COUNTER_QUERY;
2325         args[1] = c->handle;
2326         args[2] = 1; /* clear */
2327         ret = flowman_cmd(fm, args, 3);
2328         if (ret) {
2329                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
2330                             ret, c->handle);
2331                 return ret;
2332         }
2333         return 0;
2334 }
2335
2336 static int
2337 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
2338                       struct enic_fm_counter **ctr)
2339 {
2340         struct enic_fm_counter *c;
2341         int ret;
2342
2343         ENICPMD_FUNC_TRACE();
2344         *ctr = NULL;
2345         if (SLIST_EMPTY(&fm->counters)) {
2346                 ret = enic_fm_more_counters(fm);
2347                 if (ret)
2348                         return rte_flow_error_set(error, -ret,
2349                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2350                                 NULL, "enic: out of counters");
2351         }
2352         c = SLIST_FIRST(&fm->counters);
2353         SLIST_REMOVE_HEAD(&fm->counters, next);
2354         *ctr = c;
2355         return 0;
2356 }
2357
2358 static int
2359 enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
2360 {
2361         uint64_t args[2];
2362         int ret = 0;
2363
2364         ENICPMD_FUNC_TRACE();
2365         RTE_ASSERT(ah->ref > 0);
2366         ah->ref--;
2367         if (ah->ref == 0) {
2368                 args[0] = FM_ACTION_FREE;
2369                 args[1] = ah->handle;
2370                 ret = flowman_cmd(fm, args, 2);
2371                 if (ret)
2372                         /* This is a "should never happen" error. */
2373                         ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
2374                                     PRIx64, ret, ah->handle);
2375                 rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
2376                 free(ah);
2377         }
2378         return ret;
2379 }
2380
2381 static int
2382 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
2383 {
2384         uint64_t args[2];
2385         int rc;
2386
2387         ENICPMD_FUNC_TRACE();
2388         args[0] = FM_MATCH_ENTRY_REMOVE;
2389         args[1] = handle;
2390         rc = flowman_cmd(fm, args, 2);
2391         if (rc)
2392                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
2393                             " handle=0x%" PRIx64, rc, handle);
2394         return rc;
2395 }
2396
2397 static struct enic_fm_jump_flow *
2398 find_jump_flow(struct enic_flowman *fm, uint32_t group)
2399 {
2400         struct enic_fm_jump_flow *j;
2401
2402         ENICPMD_FUNC_TRACE();
2403         TAILQ_FOREACH(j, &fm->jump_list, list) {
2404                 if (j->group == group)
2405                         return j;
2406         }
2407         return NULL;
2408 }
2409
2410 static void
2411 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2412 {
2413         struct enic_fm_jump_flow *j;
2414
2415         ENICPMD_FUNC_TRACE();
2416         TAILQ_FOREACH(j, &fm->jump_list, list) {
2417                 if (j->flow == flow) {
2418                         TAILQ_REMOVE(&fm->jump_list, j, list);
2419                         free(j);
2420                         return;
2421                 }
2422         }
2423 }
2424
2425 static int
2426 save_jump_flow(struct enic_flowman *fm,
2427                struct rte_flow *flow,
2428                uint32_t group,
2429                struct fm_tcam_match_entry *match,
2430                struct fm_action *action)
2431 {
2432         struct enic_fm_jump_flow *j;
2433
2434         ENICPMD_FUNC_TRACE();
2435         j = calloc(1, sizeof(struct enic_fm_jump_flow));
2436         if (j == NULL)
2437                 return -ENOMEM;
2438         j->flow = flow;
2439         j->group = group;
2440         j->match = *match;
2441         j->action = *action;
2442         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2443         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2444         return 0;
2445 }
2446
2447 static void
2448 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2449 {
2450         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2451                 enic_fm_entry_free(fm, fm_flow->entry_handle);
2452                 fm_flow->entry_handle = FM_INVALID_HANDLE;
2453         }
2454         if (fm_flow->action != NULL) {
2455                 enic_fm_action_free(fm, fm_flow->action);
2456                 fm_flow->action = NULL;
2457         }
2458         enic_fm_counter_free(fm, fm_flow);
2459         if (fm_flow->fet) {
2460                 enic_fet_put(fm, fm_flow->fet);
2461                 fm_flow->fet = NULL;
2462         }
2463 }
2464
2465 static void
2466 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2467 {
2468         struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2469
2470         if (flow->fm->fet && flow->fm->fet->default_key)
2471                 remove_jump_flow(fm, flow);
2472         __enic_fm_flow_free(fm, flow->fm);
2473         if (steer) {
2474                 __enic_fm_flow_free(fm, steer);
2475                 free(steer);
2476         }
2477         free(flow->fm);
2478         free(flow);
2479 }
2480
2481 static int
2482 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2483                        struct fm_tcam_match_entry *match_in,
2484                        uint64_t *entry_handle,
2485                        uint8_t ingress,
2486                        struct rte_flow_error *error)
2487 {
2488         struct fm_tcam_match_entry *ftm;
2489         uint64_t args[3];
2490         int ret;
2491
2492         ENICPMD_FUNC_TRACE();
2493         /* Copy entry to the command buffer */
2494         ftm = &fm->cmd.va->fm_tcam_match_entry;
2495         memcpy(ftm, match_in, sizeof(*ftm));
2496         /* Add TCAM entry */
2497         args[0] = FM_TCAM_ENTRY_INSTALL;
2498         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2499         args[2] = fm->cmd.pa;
2500         ret = flowman_cmd(fm, args, 3);
2501         if (ret != 0) {
2502                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2503                             ingress ? "ingress" : "egress", ret);
2504                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2505                         NULL, "enic: devcmd(tcam-entry-install)");
2506                 return ret;
2507         }
2508         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2509                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
2510         *entry_handle = args[0];
2511         return 0;
2512 }
2513
2514 static int
2515 enic_fm_add_exact_entry(struct enic_flowman *fm,
2516                         struct fm_tcam_match_entry *match_in,
2517                         uint64_t *entry_handle,
2518                         struct enic_fm_fet *fet,
2519                         struct rte_flow_error *error)
2520 {
2521         struct fm_exact_match_entry *fem;
2522         uint64_t args[3];
2523         int ret;
2524
2525         ENICPMD_FUNC_TRACE();
2526         /* The new entry must have the table's key */
2527         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2528                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2529                 return rte_flow_error_set(error, EINVAL,
2530                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2531                         "enic: key does not match group's key");
2532         }
2533
2534         /* Copy entry to the command buffer */
2535         fem = &fm->cmd.va->fm_exact_match_entry;
2536         /*
2537          * Translate TCAM entry to exact entry. As is only need to drop
2538          * position and mask. The mask is part of the exact match table.
2539          * Position (aka priority) is not supported in the exact match table.
2540          */
2541         fem->fem_data = match_in->ftm_data;
2542         fem->fem_flags = match_in->ftm_flags;
2543         fem->fem_action = match_in->ftm_action;
2544         fem->fem_counter = match_in->ftm_counter;
2545
2546         /* Add exact entry */
2547         args[0] = FM_EXACT_ENTRY_INSTALL;
2548         args[1] = fet->handle;
2549         args[2] = fm->cmd.pa;
2550         ret = flowman_cmd(fm, args, 3);
2551         if (ret != 0) {
2552                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2553                             fet->ingress ? "ingress" : "egress", fet->group);
2554                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2555                         NULL, "enic: devcmd(exact-entry-install)");
2556                 return ret;
2557         }
2558         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2559                     " handle=0x%" PRIx64,
2560                     fet->ingress ? "ingress" : "egress", fet->group,
2561                     (uint64_t)args[0]);
2562         *entry_handle = args[0];
2563         return 0;
2564 }
2565
2566 static int
2567 enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
2568                        struct rte_flow_error *error,
2569                        struct enic_fm_action **ah_o)
2570 {
2571         struct enic_fm_action *ah;
2572         struct fm_action *fma;
2573         uint64_t args[2];
2574         int ret = 0;
2575
2576         ret = rte_hash_lookup_data(fm->action_hash, action_in,
2577                                    (void **)&ah);
2578         if (ret < 0 && ret != -ENOENT)
2579                 return rte_flow_error_set(error, -ret,
2580                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2581                                    NULL, "enic: rte_hash_lookup(action)");
2582
2583         if (ret == -ENOENT) {
2584                 /* Allocate a new action on the NIC. */
2585                 fma = &fm->cmd.va->fm_action;
2586                 memcpy(fma, action_in, sizeof(*fma));
2587
2588                 ah = calloc(1, sizeof(*ah));
2589                 if (ah == NULL)
2590                         return rte_flow_error_set(error, ENOMEM,
2591                                            RTE_FLOW_ERROR_TYPE_HANDLE,
2592                                            NULL, "enic: calloc(fm-action)");
2593                 memcpy(&ah->key, action_in, sizeof(struct fm_action));
2594                 args[0] = FM_ACTION_ALLOC;
2595                 args[1] = fm->cmd.pa;
2596                 ret = flowman_cmd(fm, args, 2);
2597                 if (ret != 0) {
2598                         rte_flow_error_set(error, -ret,
2599                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2600                                            NULL, "enic: devcmd(action-alloc)");
2601                         goto error_with_ah;
2602                 }
2603                 ah->handle = args[0];
2604                 ret = rte_hash_add_key_data(fm->action_hash,
2605                                             (const void *)action_in,
2606                                             (void *)ah);
2607                 if (ret != 0) {
2608                         rte_flow_error_set(error, -ret,
2609                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2610                                            NULL,
2611                                            "enic: rte_hash_add_key_data(actn)");
2612                         goto error_with_action_handle;
2613                 }
2614                 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
2615                             ah->handle);
2616         }
2617
2618         /* Action handle struct is valid, increment reference count. */
2619         ah->ref++;
2620         *ah_o = ah;
2621         return 0;
2622 error_with_action_handle:
2623         args[0] = FM_ACTION_FREE;
2624         args[1] = ah->handle;
2625         ret = flowman_cmd(fm, args, 2);
2626         if (ret != 0)
2627                 rte_flow_error_set(error, -ret,
2628                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2629                                    NULL, "enic: devcmd(action-free)");
2630 error_with_ah:
2631         free(ah);
2632         return ret;
2633 }
2634
2635 /* Push match-action to the NIC. */
2636 static int
2637 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2638                          struct enic_fm_flow *fm_flow,
2639                          struct fm_tcam_match_entry *match_in,
2640                          struct fm_action *action_in,
2641                          uint32_t group,
2642                          uint8_t ingress,
2643                          struct rte_flow_error *error)
2644 {
2645         struct enic_fm_counter *ctr;
2646         struct enic_fm_action *ah = NULL;
2647         uint64_t entry_h;
2648         int ret;
2649
2650         ENICPMD_FUNC_TRACE();
2651
2652         /* Get or create an action handle. */
2653         ret = enic_action_handle_get(fm, action_in, error, &ah);
2654         if (ret)
2655                 return ret;
2656         match_in->ftm_action = ah->handle;
2657         fm_flow->action = ah;
2658
2659         /* Allocate counter if requested. */
2660         if (match_in->ftm_flags & FMEF_COUNTER) {
2661                 ret = enic_fm_counter_alloc(fm, error, &ctr);
2662                 if (ret) /* error has been filled in */
2663                         return ret;
2664                 fm_flow->counter_valid = true;
2665                 fm_flow->counter = ctr;
2666                 match_in->ftm_counter = ctr->handle;
2667         }
2668
2669         /*
2670          * Get the group's table (either TCAM or exact match table) and
2671          * add entry to it. If we use the exact match table, the handler
2672          * will translate the TCAM entry (match_in) to the appropriate
2673          * exact match entry and use that instead.
2674          */
2675         entry_h = FM_INVALID_HANDLE;
2676         if (group == FM_TCAM_RTE_GROUP) {
2677                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2678                                              error);
2679                 if (ret)
2680                         return ret;
2681                 /* Jump action might have a ref to fet */
2682                 fm_flow->fet = fm->fet;
2683                 fm->fet = NULL;
2684         } else {
2685                 struct enic_fm_fet *fet = NULL;
2686
2687                 ret = enic_fet_get(fm, group, ingress,
2688                                    &match_in->ftm_mask, &fet, error);
2689                 if (ret)
2690                         return ret;
2691                 fm_flow->fet = fet;
2692                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2693                                               error);
2694                 if (ret)
2695                         return ret;
2696         }
2697         /* Clear counter after adding entry, as it requires in-use counter */
2698         if (fm_flow->counter_valid) {
2699                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2700                 if (ret)
2701                         return ret;
2702         }
2703         fm_flow->entry_handle = entry_h;
2704         return 0;
2705 }
2706
2707 /* Push match-action to the NIC. */
2708 static struct rte_flow *
2709 enic_fm_flow_add_entry(struct enic_flowman *fm,
2710                        struct fm_tcam_match_entry *match_in,
2711                        struct fm_action *action_in,
2712                        const struct rte_flow_attr *attrs,
2713                        struct rte_flow_error *error)
2714 {
2715         struct enic_fm_flow *fm_flow;
2716         struct rte_flow *flow;
2717
2718         ENICPMD_FUNC_TRACE();
2719         match_in->ftm_position = attrs->priority;
2720         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2721         flow = calloc(1, sizeof(*flow));
2722         fm_flow = calloc(1, sizeof(*fm_flow));
2723         if (flow == NULL || fm_flow == NULL) {
2724                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2725                         NULL, "enic: cannot allocate rte_flow");
2726                 free(flow);
2727                 free(fm_flow);
2728                 return NULL;
2729         }
2730         flow->fm = fm_flow;
2731         fm_flow->action = NULL;
2732         fm_flow->entry_handle = FM_INVALID_HANDLE;
2733         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2734                                      attrs->group, attrs->ingress, error)) {
2735                 enic_fm_flow_free(fm, flow);
2736                 return NULL;
2737         }
2738         return flow;
2739 }
2740
2741 static void
2742 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2743                    struct rte_flow_error *error)
2744 {
2745         struct enic_fm_flow *fm_flow;
2746         struct enic_fm_jump_flow *j;
2747         struct fm_action *fma;
2748         uint32_t group;
2749
2750         ENICPMD_FUNC_TRACE();
2751         /*
2752          * Find the saved flows that should jump to the new table (fet).
2753          * Then delete the old TCAM entry that jumps to the default table,
2754          * and add a new one that jumps to the new table.
2755          */
2756         group = fet->group;
2757         j = find_jump_flow(fm, group);
2758         while (j) {
2759                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2760                             j->flow, group);
2761                 /* Delete old entry */
2762                 fm_flow = j->flow->fm;
2763                 __enic_fm_flow_free(fm, fm_flow);
2764
2765                 /* Add new entry */
2766                 fma = &j->action;
2767                 fma->fma_action_ops[0].exact.handle = fet->handle;
2768                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2769                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2770                         /* Cannot roll back changes at the moment */
2771                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2772                                     j->flow);
2773                 } else {
2774                         fm_flow->fet = fet;
2775                         fet->ref++;
2776                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2777                                     fet->group, fet->ref);
2778                 }
2779
2780                 TAILQ_REMOVE(&fm->jump_list, j, list);
2781                 free(j);
2782                 j = find_jump_flow(fm, group);
2783         }
2784 }
2785
2786 static int
2787 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2788                   struct rte_flow_error *error)
2789 {
2790         struct fm_tcam_match_entry *fm_tcam_entry;
2791         struct enic_fm_flow *fm_flow;
2792         struct fm_action *fm_action;
2793         struct fm_action_op fm_op;
2794         int ret;
2795
2796         ENICPMD_FUNC_TRACE();
2797         fm_flow = calloc(1, sizeof(*fm_flow));
2798         if (fm_flow == NULL) {
2799                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2800                         NULL, "enic: cannot allocate rte_flow");
2801                 return -ENOMEM;
2802         }
2803         /* Original egress hairpin flow */
2804         fm_tcam_entry = &fm->tcam_entry;
2805         fm_action = &fm->action;
2806         /* Use the match pattern of the egress flow as is, without counters */
2807         fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2808         /* The only action is steer to vnic */
2809         fm->action_op_count = 0;
2810         memset(fm_action, 0, sizeof(*fm_action));
2811         memset(&fm_op, 0, sizeof(fm_op));
2812         /* Always to queue 0 for now */
2813         fm_op.fa_op = FMOP_RQ_STEER;
2814         fm_op.rq_steer.rq_index = 0;
2815         fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2816         ret = enic_fm_append_action_op(fm, &fm_op, error);
2817         if (ret)
2818                 goto error_with_flow;
2819         ENICPMD_LOG(DEBUG, "add steer op");
2820         /* Add required END */
2821         memset(&fm_op, 0, sizeof(fm_op));
2822         fm_op.fa_op = FMOP_END;
2823         ret = enic_fm_append_action_op(fm, &fm_op, error);
2824         if (ret)
2825                 goto error_with_flow;
2826         /* Add the ingress flow */
2827         fm_flow->action = NULL;
2828         fm_flow->entry_handle = FM_INVALID_HANDLE;
2829         ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2830                                        FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2831         if (ret) {
2832                 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2833                 goto error_with_flow;
2834         }
2835         /* The new flow is now the egress flow's paired flow */
2836         flow->fm->hairpin_steer_flow = fm_flow;
2837         return 0;
2838
2839 error_with_flow:
2840         free(fm_flow);
2841         return ret;
2842 }
2843
2844 static void
2845 enic_fm_open_scratch(struct enic_flowman *fm)
2846 {
2847         fm->action_op_count = 0;
2848         fm->fet = NULL;
2849         fm->need_hairpin_steer = 0;
2850         fm->hairpin_steer_vnic_h = 0;
2851         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2852         memset(&fm->action, 0, sizeof(fm->action));
2853 }
2854
2855 static void
2856 enic_fm_close_scratch(struct enic_flowman *fm)
2857 {
2858         if (fm->fet) {
2859                 enic_fet_put(fm, fm->fet);
2860                 fm->fet = NULL;
2861         }
2862         fm->action_op_count = 0;
2863 }
2864
2865 static int
2866 enic_fm_flow_validate(struct rte_eth_dev *dev,
2867                       const struct rte_flow_attr *attrs,
2868                       const struct rte_flow_item pattern[],
2869                       const struct rte_flow_action actions[],
2870                       struct rte_flow_error *error)
2871 {
2872         struct fm_tcam_match_entry *fm_tcam_entry;
2873         struct fm_action *fm_action;
2874         struct enic_flowman *fm;
2875         int ret;
2876
2877         ENICPMD_FUNC_TRACE();
2878         fm = begin_fm(pmd_priv(dev));
2879         if (fm == NULL)
2880                 return -ENOTSUP;
2881         enic_fm_open_scratch(fm);
2882         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2883         if (!ret) {
2884                 fm_tcam_entry = &fm->tcam_entry;
2885                 fm_action = &fm->action;
2886                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2887                                         attrs->ingress);
2888         }
2889         enic_fm_close_scratch(fm);
2890         end_fm(fm);
2891         return ret;
2892 }
2893
2894 static int
2895 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2896                          struct rte_flow *flow, void *data,
2897                          struct rte_flow_error *error)
2898 {
2899         struct rte_flow_query_count *query;
2900         struct enic_fm_flow *fm_flow;
2901         struct enic_flowman *fm;
2902         uint64_t args[3];
2903         int rc;
2904
2905         ENICPMD_FUNC_TRACE();
2906         fm = begin_fm(pmd_priv(dev));
2907         query = data;
2908         fm_flow = flow->fm;
2909         if (!fm_flow->counter_valid) {
2910                 rc = rte_flow_error_set(error, ENOTSUP,
2911                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2912                         "enic: flow does not have counter");
2913                 goto exit;
2914         }
2915
2916         args[0] = FM_COUNTER_QUERY;
2917         args[1] = fm_flow->counter->handle;
2918         args[2] = query->reset;
2919         rc = flowman_cmd(fm, args, 3);
2920         if (rc) {
2921                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2922                             rc, fm_flow->counter->handle);
2923                 goto exit;
2924         }
2925         query->hits_set = 1;
2926         query->hits = args[0];
2927         query->bytes_set = 1;
2928         query->bytes = args[1];
2929         rc = 0;
2930 exit:
2931         end_fm(fm);
2932         return rc;
2933 }
2934
2935 static int
2936 enic_fm_flow_query(struct rte_eth_dev *dev,
2937                    struct rte_flow *flow,
2938                    const struct rte_flow_action *actions,
2939                    void *data,
2940                    struct rte_flow_error *error)
2941 {
2942         int ret = 0;
2943
2944         ENICPMD_FUNC_TRACE();
2945         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2946                 switch (actions->type) {
2947                 case RTE_FLOW_ACTION_TYPE_VOID:
2948                         break;
2949                 case RTE_FLOW_ACTION_TYPE_COUNT:
2950                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2951                         break;
2952                 default:
2953                         return rte_flow_error_set(error, ENOTSUP,
2954                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2955                                                   actions,
2956                                                   "action not supported");
2957                 }
2958                 if (ret < 0)
2959                         return ret;
2960         }
2961         return 0;
2962 }
2963
2964 static struct rte_flow *
2965 enic_fm_flow_create(struct rte_eth_dev *dev,
2966                     const struct rte_flow_attr *attrs,
2967                     const struct rte_flow_item pattern[],
2968                     const struct rte_flow_action actions[],
2969                     struct rte_flow_error *error)
2970 {
2971         struct fm_tcam_match_entry *fm_tcam_entry;
2972         struct fm_action *fm_action;
2973         struct enic_flowman *fm;
2974         struct enic_fm_fet *fet;
2975         struct rte_flow *flow;
2976         struct enic *enic;
2977         int ret;
2978
2979         ENICPMD_FUNC_TRACE();
2980         enic = pmd_priv(dev);
2981         fm = begin_fm(enic);
2982         if (fm == NULL) {
2983                 rte_flow_error_set(error, ENOTSUP,
2984                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2985                         "flowman is not initialized");
2986                 return NULL;
2987         }
2988         enic_fm_open_scratch(fm);
2989         flow = NULL;
2990         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2991         if (ret < 0)
2992                 goto error_with_scratch;
2993         fm_tcam_entry = &fm->tcam_entry;
2994         fm_action = &fm->action;
2995         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2996                                       attrs, error);
2997         if (flow) {
2998                 /* Add ingress rule that pairs with hairpin rule */
2999                 if (fm->need_hairpin_steer) {
3000                         ret = add_hairpin_steer(fm, flow, error);
3001                         if (ret) {
3002                                 enic_fm_flow_free(fm, flow);
3003                                 flow = NULL;
3004                                 goto error_with_scratch;
3005                         }
3006                 }
3007                 LIST_INSERT_HEAD(&enic->flows, flow, next);
3008                 fet = flow->fm->fet;
3009                 if (fet && fet->default_key) {
3010                         /*
3011                          * Jump to non-existent group? Save the relevant info
3012                          * so we can convert this flow when that group
3013                          * materializes.
3014                          */
3015                         save_jump_flow(fm, flow, fet->group,
3016                                        fm_tcam_entry, fm_action);
3017                 } else if (fet && fet->ref == 1) {
3018                         /*
3019                          * A new table is created. Convert the saved flows
3020                          * that should jump to this group.
3021                          */
3022                         convert_jump_flows(fm, fet, error);
3023                 }
3024         }
3025
3026 error_with_scratch:
3027         enic_fm_close_scratch(fm);
3028         end_fm(fm);
3029         return flow;
3030 }
3031
3032 static int
3033 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
3034                      __rte_unused struct rte_flow_error *error)
3035 {
3036         struct enic *enic = pmd_priv(dev);
3037         struct enic_flowman *fm;
3038
3039         ENICPMD_FUNC_TRACE();
3040         fm = begin_fm(enic);
3041         if (fm == NULL)
3042                 return 0;
3043         LIST_REMOVE(flow, next);
3044         enic_fm_flow_free(fm, flow);
3045         end_fm(fm);
3046         return 0;
3047 }
3048
3049 static int
3050 enic_fm_flow_flush(struct rte_eth_dev *dev,
3051                    __rte_unused struct rte_flow_error *error)
3052 {
3053         LIST_HEAD(enic_flows, rte_flow) internal;
3054         struct enic_fm_flow *fm_flow;
3055         struct enic_flowman *fm;
3056         struct rte_flow *flow;
3057         struct enic *enic = pmd_priv(dev);
3058
3059         ENICPMD_FUNC_TRACE();
3060
3061         fm = begin_fm(enic);
3062         if (fm == NULL)
3063                 return 0;
3064         /* Destroy all non-internal flows */
3065         LIST_INIT(&internal);
3066         while (!LIST_EMPTY(&enic->flows)) {
3067                 flow = LIST_FIRST(&enic->flows);
3068                 fm_flow = flow->fm;
3069                 LIST_REMOVE(flow, next);
3070                 if (flow->internal) {
3071                         LIST_INSERT_HEAD(&internal, flow, next);
3072                         continue;
3073                 }
3074                 /*
3075                  * If tables are null, then vNIC is closing, and the firmware
3076                  * has already cleaned up flowman state. So do not try to free
3077                  * resources, as it only causes errors.
3078                  */
3079                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
3080                         fm_flow->entry_handle = FM_INVALID_HANDLE;
3081                         fm_flow->action = NULL;
3082                         fm_flow->fet = NULL;
3083                 }
3084                 enic_fm_flow_free(fm, flow);
3085         }
3086         while (!LIST_EMPTY(&internal)) {
3087                 flow = LIST_FIRST(&internal);
3088                 LIST_REMOVE(flow, next);
3089                 LIST_INSERT_HEAD(&enic->flows, flow, next);
3090         }
3091         end_fm(fm);
3092         return 0;
3093 }
3094
3095 static int
3096 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
3097 {
3098         uint64_t args[2];
3099         int rc;
3100
3101         args[0] = FM_MATCH_TABLE_FREE;
3102         args[1] = handle;
3103         rc = flowman_cmd(fm, args, 2);
3104         if (rc)
3105                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
3106                             rc, handle);
3107         return rc;
3108 }
3109
3110 static int
3111 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
3112                         uint32_t max_entries, uint64_t *handle)
3113 {
3114         struct fm_tcam_match_table *tcam_tbl;
3115         uint64_t args[2];
3116         int rc;
3117
3118         ENICPMD_FUNC_TRACE();
3119         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
3120         tcam_tbl->ftt_direction = direction;
3121         tcam_tbl->ftt_stage = FM_STAGE_LAST;
3122         tcam_tbl->ftt_max_entries = max_entries;
3123         args[0] = FM_TCAM_TABLE_ALLOC;
3124         args[1] = fm->cmd.pa;
3125         rc = flowman_cmd(fm, args, 2);
3126         if (rc) {
3127                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
3128                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
3129                 return rc;
3130         }
3131         *handle = args[0];
3132         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
3133                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
3134         return 0;
3135 }
3136
3137 static int
3138 enic_fm_init_actions(struct enic_flowman *fm)
3139 {
3140         struct rte_hash *a_hash;
3141         char name[RTE_HASH_NAMESIZE];
3142         struct rte_hash_parameters params = {
3143                 .entries = FM_MAX_ACTION_TABLE_SIZE,
3144                 .key_len = sizeof(struct fm_action),
3145                 .hash_func = rte_jhash,
3146                 .hash_func_init_val = 0,
3147                 .socket_id = rte_socket_id(),
3148         };
3149
3150         ENICPMD_FUNC_TRACE();
3151         snprintf((char *)name, sizeof(name), "fm-ah-%s",
3152                  fm->owner_enic->bdf_name);
3153         params.name = name;
3154
3155         a_hash = rte_hash_create(&params);
3156         if (a_hash == NULL)
3157                 return -rte_errno;
3158         fm->action_hash = a_hash;
3159         return 0;
3160 }
3161
3162 static int
3163 enic_fm_init_counters(struct enic_flowman *fm)
3164 {
3165         ENICPMD_FUNC_TRACE();
3166         SLIST_INIT(&fm->counters);
3167         return enic_fm_more_counters(fm);
3168 }
3169
3170 static void
3171 enic_fm_free_all_counters(struct enic_flowman *fm)
3172 {
3173         uint64_t args[2];
3174         int rc;
3175
3176         args[0] = FM_COUNTER_BRK;
3177         args[1] = 0;
3178         rc = flowman_cmd(fm, args, 2);
3179         if (rc != 0)
3180                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
3181         rte_free(fm->counter_stack);
3182 }
3183
3184 static int
3185 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
3186 {
3187         int rc;
3188
3189         ENICPMD_FUNC_TRACE();
3190         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
3191                                     &fm->ig_tcam_hndl);
3192         if (rc)
3193                 return rc;
3194         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
3195                                     &fm->eg_tcam_hndl);
3196         return rc;
3197 }
3198
3199 static void
3200 enic_fm_free_tcam_tables(struct enic_flowman *fm)
3201 {
3202         ENICPMD_FUNC_TRACE();
3203         if (fm->ig_tcam_hndl) {
3204                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
3205                             fm->ig_tcam_hndl);
3206                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
3207                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
3208         }
3209         if (fm->eg_tcam_hndl) {
3210                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
3211                             fm->eg_tcam_hndl);
3212                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
3213                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
3214         }
3215 }
3216
3217 int
3218 enic_fm_init(struct enic *enic)
3219 {
3220         const struct rte_pci_addr *addr;
3221         struct enic_flowman *fm;
3222         uint8_t name[RTE_MEMZONE_NAMESIZE];
3223         int rc;
3224
3225         if (enic->flow_filter_mode != FILTER_FLOWMAN)
3226                 return 0;
3227         ENICPMD_FUNC_TRACE();
3228         /* Get vnic handle and save for port-id action */
3229         if (enic_is_vf_rep(enic))
3230                 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
3231         else
3232                 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
3233         rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
3234         if (rc) {
3235                 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
3236                             addr->bus, addr->devid, addr->function);
3237                 return rc;
3238         }
3239         /* Save UIF for egport action */
3240         enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
3241         ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
3242         /* Nothing else to do for representor. It will share the PF flowman */
3243         if (enic_is_vf_rep(enic))
3244                 return 0;
3245         fm = calloc(1, sizeof(*fm));
3246         if (fm == NULL) {
3247                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
3248                 return -ENOMEM;
3249         }
3250         fm->owner_enic = enic;
3251         rte_spinlock_init(&fm->lock);
3252         TAILQ_INIT(&fm->fet_list);
3253         TAILQ_INIT(&fm->jump_list);
3254         /* Allocate host memory for flowman commands */
3255         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
3256         fm->cmd.va = enic_alloc_consistent(enic,
3257                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
3258         if (!fm->cmd.va) {
3259                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
3260                 rc = -ENOMEM;
3261                 goto error_fm;
3262         }
3263         /* Allocate TCAM tables upfront as they are the main tables */
3264         rc = enic_fm_alloc_tcam_tables(fm);
3265         if (rc) {
3266                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
3267                 goto error_cmd;
3268         }
3269         /* Then a number of counters */
3270         rc = enic_fm_init_counters(fm);
3271         if (rc) {
3272                 ENICPMD_LOG(ERR, "cannot alloc counters");
3273                 goto error_tables;
3274         }
3275         /* set up action handle hash */
3276         rc = enic_fm_init_actions(fm);
3277         if (rc) {
3278                 ENICPMD_LOG(ERR, "cannot create action hash, error:%d", rc);
3279                 goto error_counters;
3280         }
3281         /*
3282          * One default exact match table for each direction. We hold onto
3283          * it until close.
3284          */
3285         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
3286         if (rc) {
3287                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
3288                 goto error_actions;
3289         }
3290         fm->default_ig_fet->ref = 1;
3291         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
3292         if (rc) {
3293                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
3294                 goto error_ig_fet;
3295         }
3296         fm->default_eg_fet->ref = 1;
3297         fm->vf_rep_tag = FM_VF_REP_TAG;
3298         enic->fm = fm;
3299         return 0;
3300
3301 error_ig_fet:
3302         enic_fet_free(fm, fm->default_ig_fet);
3303 error_actions:
3304         rte_hash_free(fm->action_hash);
3305 error_counters:
3306         enic_fm_free_all_counters(fm);
3307 error_tables:
3308         enic_fm_free_tcam_tables(fm);
3309 error_cmd:
3310         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3311                 fm->cmd.va, fm->cmd.pa);
3312 error_fm:
3313         free(fm);
3314         return rc;
3315 }
3316
3317 void
3318 enic_fm_destroy(struct enic *enic)
3319 {
3320         struct enic_flowman *fm;
3321         struct enic_fm_fet *fet;
3322
3323         ENICPMD_FUNC_TRACE();
3324         if (enic_is_vf_rep(enic)) {
3325                 delete_rep_flows(enic);
3326                 return;
3327         }
3328         if (enic->fm == NULL)
3329                 return;
3330         fm = enic->fm;
3331         enic_fm_flow_flush(enic->rte_dev, NULL);
3332         enic_fet_free(fm, fm->default_eg_fet);
3333         enic_fet_free(fm, fm->default_ig_fet);
3334         /* Free all exact match tables still open */
3335         while (!TAILQ_EMPTY(&fm->fet_list)) {
3336                 fet = TAILQ_FIRST(&fm->fet_list);
3337                 enic_fet_free(fm, fet);
3338         }
3339         enic_fm_free_tcam_tables(fm);
3340         enic_fm_free_all_counters(fm);
3341         rte_hash_free(fm->action_hash);
3342         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3343                 fm->cmd.va, fm->cmd.pa);
3344         fm->cmd.va = NULL;
3345         free(fm);
3346         enic->fm = NULL;
3347 }
3348
3349 int
3350 enic_fm_allocate_switch_domain(struct enic *pf)
3351 {
3352         const struct rte_pci_addr *cur_a, *prev_a;
3353         struct rte_eth_dev *dev;
3354         struct enic *cur, *prev;
3355         uint16_t domain_id;
3356         uint64_t vnic_h;
3357         uint16_t pid;
3358         int ret;
3359
3360         ENICPMD_FUNC_TRACE();
3361         if (enic_is_vf_rep(pf))
3362                 return -EINVAL;
3363         cur = pf;
3364         cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
3365         /* Go through ports and find another PF that is on the same adapter */
3366         RTE_ETH_FOREACH_DEV(pid) {
3367                 dev = &rte_eth_devices[pid];
3368                 if (!dev_is_enic(dev))
3369                         continue;
3370                 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
3371                         continue;
3372                 if (dev == cur->rte_dev)
3373                         continue;
3374                 /* dev is another PF. Is it on the same adapter? */
3375                 prev = pmd_priv(dev);
3376                 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
3377                 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
3378                         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
3379                                 cur->rte_dev->data->port_id,
3380                                 cur_a->bus, cur_a->devid, cur_a->function,
3381                                 dev->data->port_id,
3382                                 prev_a->bus, prev_a->devid, prev_a->function,
3383                                 prev->switch_domain_id);
3384                         cur->switch_domain_id = prev->switch_domain_id;
3385                         return 0;
3386                 }
3387         }
3388         ret = rte_eth_switch_domain_alloc(&domain_id);
3389         if (ret) {
3390                 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
3391                             ret);
3392         }
3393         cur->switch_domain_id = domain_id;
3394         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
3395                     cur->rte_dev->data->port_id,
3396                     cur_a->bus, cur_a->devid, cur_a->function,
3397                     domain_id);
3398         return ret;
3399 }
3400
3401 const struct rte_flow_ops enic_fm_flow_ops = {
3402         .validate = enic_fm_flow_validate,
3403         .create = enic_fm_flow_create,
3404         .destroy = enic_fm_flow_destroy,
3405         .flush = enic_fm_flow_flush,
3406         .query = enic_fm_flow_query,
3407 };
3408
3409 /* Add a high priority flow that loops representor packets to VF */
3410 int
3411 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
3412 {
3413         struct fm_tcam_match_entry *fm_tcam_entry;
3414         struct rte_flow *flow0, *flow1;
3415         struct fm_action *fm_action;
3416         struct rte_flow_error error;
3417         struct rte_flow_attr attrs;
3418         struct fm_action_op fm_op;
3419         struct enic_flowman *fm;
3420         struct enic *pf;
3421         uint8_t tag;
3422
3423         pf = vf->pf;
3424         fm = pf->fm;
3425         tag = fm->vf_rep_tag;
3426         enic_fm_open_scratch(fm);
3427         fm_tcam_entry = &fm->tcam_entry;
3428         fm_action = &fm->action;
3429         /* Egress rule: match WQ ID and tag+hairpin */
3430         fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
3431         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3432         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3433         memset(&fm_op, 0, sizeof(fm_op));
3434         fm_op.fa_op = FMOP_TAG;
3435         fm_op.tag.tag = tag;
3436         enic_fm_append_action_op(fm, &fm_op, &error);
3437         memset(&fm_op, 0, sizeof(fm_op));
3438         fm_op.fa_op = FMOP_EG_HAIRPIN;
3439         enic_fm_append_action_op(fm, &fm_op, &error);
3440         memset(&fm_op, 0, sizeof(fm_op));
3441         fm_op.fa_op = FMOP_END;
3442         enic_fm_append_action_op(fm, &fm_op, &error);
3443         attrs.group = 0;
3444         attrs.ingress = 0;
3445         attrs.egress = 1;
3446         attrs.priority = FM_HIGHEST_PRIORITY;
3447         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3448                                        &attrs, &error);
3449         enic_fm_close_scratch(fm);
3450         if (flow0 == NULL) {
3451                 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
3452                 return -EINVAL;
3453         }
3454         LIST_INSERT_HEAD(&pf->flows, flow0, next);
3455         /* Make this flow internal, so the user app cannot delete it */
3456         flow0->internal = 1;
3457         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
3458                     vf->vf_id, vf->pf_wq_idx, tag);
3459
3460         /* Ingress: steer hairpinned to VF RQ 0 */
3461         enic_fm_open_scratch(fm);
3462         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3463         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3464         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3465         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3466         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3467         memset(&fm_op, 0, sizeof(fm_op));
3468         fm_op.fa_op = FMOP_RQ_STEER;
3469         fm_op.rq_steer.rq_index = 0;
3470         fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
3471         enic_fm_append_action_op(fm, &fm_op, &error);
3472         memset(&fm_op, 0, sizeof(fm_op));
3473         fm_op.fa_op = FMOP_END;
3474         enic_fm_append_action_op(fm, &fm_op, &error);
3475         attrs.group = 0;
3476         attrs.ingress = 1;
3477         attrs.egress = 0;
3478         attrs.priority = FM_HIGHEST_PRIORITY;
3479         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3480                                        &attrs, &error);
3481         enic_fm_close_scratch(fm);
3482         if (flow1 == NULL) {
3483                 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
3484                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3485                 return -EINVAL;
3486         }
3487         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3488         flow1->internal = 1;
3489         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
3490                     vf->vf_id, tag, fm_op.rq_steer.rq_index);
3491         vf->rep2vf_flow[0] = flow0;
3492         vf->rep2vf_flow[1] = flow1;
3493         /* Done with this tag, use a different one next time */
3494         fm->vf_rep_tag++;
3495         return 0;
3496 }
3497
3498 /*
3499  * Add a low priority flow that matches all packets from VF and loops them
3500  * back to the representor.
3501  */
3502 int
3503 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3504 {
3505         struct fm_tcam_match_entry *fm_tcam_entry;
3506         struct rte_flow *flow0, *flow1;
3507         struct fm_action *fm_action;
3508         struct rte_flow_error error;
3509         struct rte_flow_attr attrs;
3510         struct fm_action_op fm_op;
3511         struct enic_flowman *fm;
3512         struct enic *pf;
3513         uint8_t tag;
3514
3515         pf = vf->pf;
3516         fm = pf->fm;
3517         tag = fm->vf_rep_tag;
3518         enic_fm_open_scratch(fm);
3519         fm_tcam_entry = &fm->tcam_entry;
3520         fm_action = &fm->action;
3521         /* Egress rule: match-any and tag+hairpin */
3522         fm_tcam_entry->ftm_data.fk_wq_id = 0;
3523         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3524         fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3525         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3526         memset(&fm_op, 0, sizeof(fm_op));
3527         fm_op.fa_op = FMOP_TAG;
3528         fm_op.tag.tag = tag;
3529         enic_fm_append_action_op(fm, &fm_op, &error);
3530         memset(&fm_op, 0, sizeof(fm_op));
3531         fm_op.fa_op = FMOP_EG_HAIRPIN;
3532         enic_fm_append_action_op(fm, &fm_op, &error);
3533         memset(&fm_op, 0, sizeof(fm_op));
3534         fm_op.fa_op = FMOP_END;
3535         enic_fm_append_action_op(fm, &fm_op, &error);
3536         attrs.group = 0;
3537         attrs.ingress = 0;
3538         attrs.egress = 1;
3539         attrs.priority = FM_LOWEST_PRIORITY;
3540         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3541                                        &attrs, &error);
3542         enic_fm_close_scratch(fm);
3543         if (flow0 == NULL) {
3544                 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3545                 return -EINVAL;
3546         }
3547         LIST_INSERT_HEAD(&pf->flows, flow0, next);
3548         /* Make this flow internal, so the user app cannot delete it */
3549         flow0->internal = 1;
3550         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3551                     vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3552
3553         /* Ingress: steer hairpinned to VF rep RQ */
3554         enic_fm_open_scratch(fm);
3555         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3556         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3557         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3558         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3559         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3560         memset(&fm_op, 0, sizeof(fm_op));
3561         fm_op.fa_op = FMOP_RQ_STEER;
3562         fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3563         fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3564         enic_fm_append_action_op(fm, &fm_op, &error);
3565         memset(&fm_op, 0, sizeof(fm_op));
3566         fm_op.fa_op = FMOP_END;
3567         enic_fm_append_action_op(fm, &fm_op, &error);
3568         attrs.group = 0;
3569         attrs.ingress = 1;
3570         attrs.egress = 0;
3571         attrs.priority = FM_HIGHEST_PRIORITY;
3572         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3573                                        &attrs, &error);
3574         enic_fm_close_scratch(fm);
3575         if (flow1 == NULL) {
3576                 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3577                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3578                 return -EINVAL;
3579         }
3580         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3581         flow1->internal = 1;
3582         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3583                     vf->vf_id, tag, vf->pf_rq_sop_idx);
3584         vf->vf2rep_flow[0] = flow0;
3585         vf->vf2rep_flow[1] = flow1;
3586         /* Done with this tag, use a different one next time */
3587         fm->vf_rep_tag++;
3588         return 0;
3589 }
3590
3591 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3592 static void
3593 delete_rep_flows(struct enic *enic)
3594 {
3595         struct enic_vf_representor *vf;
3596         struct rte_flow_error error;
3597         struct rte_eth_dev *dev;
3598         uint32_t i;
3599
3600         RTE_ASSERT(enic_is_vf_rep(enic));
3601         vf = VF_ENIC_TO_VF_REP(enic);
3602         dev = vf->pf->rte_dev;
3603         for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3604                 if (vf->vf2rep_flow[i])
3605                         enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3606         }
3607         for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3608                 if (vf->rep2vf_flow[i])
3609                         enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3610         }
3611 }
3612
3613 static struct enic_flowman *
3614 begin_fm(struct enic *enic)
3615 {
3616         struct enic_vf_representor *vf;
3617         struct enic_flowman *fm;
3618
3619         /* Representor uses PF flowman */
3620         if (enic_is_vf_rep(enic)) {
3621                 vf = VF_ENIC_TO_VF_REP(enic);
3622                 fm = vf->pf->fm;
3623         } else {
3624                 fm = enic->fm;
3625         }
3626         /* Save the API caller and lock if representors exist */
3627         if (fm) {
3628                 if (fm->owner_enic->switchdev_mode)
3629                         rte_spinlock_lock(&fm->lock);
3630                 fm->user_enic = enic;
3631         }
3632         return fm;
3633 }
3634
3635 static void
3636 end_fm(struct enic_flowman *fm)
3637 {
3638         fm->user_enic = NULL;
3639         if (fm->owner_enic->switchdev_mode)
3640                 rte_spinlock_unlock(&fm->lock);
3641 }