drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_geneve.h>
  12 #include <rte_hash.h>
  13 #include <rte_jhash.h>
  14 #include <rte_ip.h>
  15 #include <rte_udp.h>
  16 #include <rte_memzone.h>
  17
  18 #include "enic_compat.h"
  19 #include "enic.h"
  20 #include "vnic_dev.h"
  21 #include "vnic_nic.h"
  22
  23 #define IP_DEFTTL  64   /* from RFC 1340. */
  24 #define IP6_VTC_FLOW 0x60000000
  25
  26 /* Up to 1024 TCAM entries */
  27 #define FM_MAX_TCAM_TABLE_SIZE 1024
  28
  29 /* Up to 4096 entries per exact match table */
  30 #define FM_MAX_EXACT_TABLE_SIZE 4096
  31
  32 /* Number of counters to increase on for each increment */
  33 #define FM_COUNTERS_EXPAND  100
  34
  35 #define FM_INVALID_HANDLE 0
  36
  37 /* Low priority used for implicit VF -> representor flow */
  38 #define FM_LOWEST_PRIORITY 100000
  39
  40 /* High priority used for implicit representor -> VF flow */
  41 #define FM_HIGHEST_PRIORITY 0
  42
  43 /* Tag used for implicit VF <-> representor flows */
  44 #define FM_VF_REP_TAG 1
  45
  46 /* Max number of actions supported by VIC is 2K. Make hash table double that. */
  47 #define FM_MAX_ACTION_TABLE_SIZE 4096
  48
  49 /*
  50  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  51  * Use a simple scheme to map groups to tables.
  52  * Group 0 uses the single TCAM tables, one for each direction.
  53  * Group 1, 2, ... uses its own exact match table.
  54  *
  55  * The TCAM tables are allocated upfront during init.
  56  *
  57  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  58  *
  59  * 1. Add a flow that jumps from group 0 to group N.
  60  *
  61  * If N does not exist, we allocate an exact match table for it, using
  62  * a dummy key. A key is required for the table.
  63  *
  64  * 2. Add a flow that uses group N.
  65  *
  66  * If N does not exist, we allocate an exact match table for it, using
  67  * the flow's key. Subsequent flows to the same group all should have
  68  * the same key.
  69  *
  70  * Without a jump flow to N, N is not reachable in hardware. No packets
  71  * reach N and match.
  72  *
  73  * 3. Add a flow to an empty group N.
  74  *
  75  * N has been created via (1) and the dummy key. We free that table, allocate
  76  * a new table using the new flow's key. Also re-do the existing jump flow to
  77  * point to the new table.
  78  */
  79 #define FM_TCAM_RTE_GROUP 0
  80
  81 struct enic_fm_fet {
  82         TAILQ_ENTRY(enic_fm_fet) list;
  83         uint32_t group; /* rte_flow group ID */
  84         uint64_t handle; /* Exact match table handle from flowman */
  85         uint8_t ingress;
  86         uint8_t default_key;
  87         int ref; /* Reference count via get/put */
  88         struct fm_key_template key; /* Key associated with the table */
  89 };
  90
  91 struct enic_fm_counter {
  92         SLIST_ENTRY(enic_fm_counter) next;
  93         uint32_t handle;
  94 };
  95
  96 struct enic_fm_action {
  97         int ref;
  98         uint64_t handle;
  99         struct fm_action key;
 100 };
 101
 102 /* rte_flow.fm */
 103 struct enic_fm_flow {
 104         bool counter_valid;
 105         uint64_t entry_handle;
 106         struct enic_fm_action  *action;
 107         struct enic_fm_counter *counter;
 108         struct enic_fm_fet *fet;
 109         /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
 110         struct enic_fm_flow *hairpin_steer_flow;
 111 };
 112
 113 struct enic_fm_jump_flow {
 114         TAILQ_ENTRY(enic_fm_jump_flow) list;
 115         struct rte_flow *flow;
 116         uint32_t group;
 117         struct fm_tcam_match_entry match;
 118         struct fm_action action;
 119 };
 120
 121 /*
 122  * Flowman uses host memory for commands. This structure is allocated
 123  * in DMA-able memory.
 124  */
 125 union enic_flowman_cmd_mem {
 126         struct fm_tcam_match_table fm_tcam_match_table;
 127         struct fm_exact_match_table fm_exact_match_table;
 128         struct fm_tcam_match_entry fm_tcam_match_entry;
 129         struct fm_exact_match_entry fm_exact_match_entry;
 130         struct fm_action fm_action;
 131 };
 132
 133 /*
 134  * PF has a flowman instance, and VF representors share it with PF.
 135  * PF allocates this structure and owns it. VF representors borrow
 136  * the PF's structure during API calls (e.g. create, query).
 137  */
 138 struct enic_flowman {
 139         struct enic *owner_enic; /* PF */
 140         struct enic *user_enic;  /* API caller (PF or representor) */
 141         /*
 142          * Representors and PF share the same underlying flowman.
 143          * Lock API calls to serialize accesses from them. Only used
 144          * when VF representors are present.
 145          */
 146         rte_spinlock_t lock;
 147         /* Command buffer */
 148         struct {
 149                 union enic_flowman_cmd_mem *va;
 150                 dma_addr_t pa;
 151         } cmd;
 152         /* TCAM tables allocated upfront, used for group 0 */
 153         uint64_t ig_tcam_hndl;
 154         uint64_t eg_tcam_hndl;
 155         /* Counters */
 156         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 157         void *counter_stack;
 158         uint32_t counters_alloced;
 159         /* Exact match tables for groups != 0, dynamically allocated */
 160         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 161         /*
 162          * Default exact match tables used for jump actions to
 163          * non-existent groups.
 164          */
 165         struct enic_fm_fet *default_eg_fet;
 166         struct enic_fm_fet *default_ig_fet;
 167         /* hash table for Action reuse */
 168         struct rte_hash *action_hash;
 169         /* Flows that jump to the default table above */
 170         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 171         /*
 172          * Scratch data used during each invocation of flow_create
 173          * and flow_validate.
 174          */
 175         struct enic_fm_fet *fet;
 176         struct fm_tcam_match_entry tcam_entry;
 177         struct fm_action action;
 178         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 179         int action_op_count;
 180         /* Tags used for representor flows */
 181         uint8_t vf_rep_tag;
 182         /* For auto-added steer action for hairpin */
 183         int need_hairpin_steer;
 184         uint64_t hairpin_steer_vnic_h;
 185 };
 186
 187 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 188 /*
 189  * API functions (create, destroy, validate, flush) call begin_fm()
 190  * upon entering to save the caller enic (PF or VF representor) and
 191  * lock. Upon exit, they call end_fm() to unlock.
 192  */
 193 static struct enic_flowman *begin_fm(struct enic *enic);
 194 static void end_fm(struct enic_flowman *fm);
 195 /* Delete internal flows created for representor paths */
 196 static void delete_rep_flows(struct enic *enic);
 197
 198 /*
 199  * Common arguments passed to copy_item functions. Use this structure
 200  * so we can easily add new arguments.
 201  * item: Item specification.
 202  * fm_tcam_entry: Flowman TCAM match entry.
 203  * header_level: 0 for outer header, 1 for inner header.
 204  */
 205 struct copy_item_args {
 206         const struct rte_flow_item *item;
 207         struct fm_tcam_match_entry *fm_tcam_entry;
 208         uint8_t header_level;
 209         struct rte_flow_error *error;
 210 };
 211
 212 /* functions for copying items into flowman match */
 213 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 214
 215 /* Info about how to copy items into flowman match */
 216 struct enic_fm_items {
 217         /* Function for copying and validating an item. */
 218         enic_copy_item_fn * const copy_item;
 219         /* List of valid previous items. */
 220         const enum rte_flow_item_type * const prev_items;
 221         /*
 222          * True if it's OK for this item to be the first item. For some NIC
 223          * versions, it's invalid to start the stack above layer 3.
 224          */
 225         const uint8_t valid_start_item;
 226 };
 227
 228 static enic_copy_item_fn enic_fm_copy_item_eth;
 229 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 230 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 231 static enic_copy_item_fn enic_fm_copy_item_raw;
 232 static enic_copy_item_fn enic_fm_copy_item_sctp;
 233 static enic_copy_item_fn enic_fm_copy_item_tcp;
 234 static enic_copy_item_fn enic_fm_copy_item_udp;
 235 static enic_copy_item_fn enic_fm_copy_item_vlan;
 236 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 237 static enic_copy_item_fn enic_fm_copy_item_gtp;
 238 static enic_copy_item_fn enic_fm_copy_item_geneve;
 239 static enic_copy_item_fn enic_fm_copy_item_geneve_opt;
 240
 241 /* Ingress actions */
 242 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 243         RTE_FLOW_ACTION_TYPE_COUNT,
 244         RTE_FLOW_ACTION_TYPE_DROP,
 245         RTE_FLOW_ACTION_TYPE_FLAG,
 246         RTE_FLOW_ACTION_TYPE_JUMP,
 247         RTE_FLOW_ACTION_TYPE_MARK,
 248         RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
 249         RTE_FLOW_ACTION_TYPE_PORT_ID,
 250         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 251         RTE_FLOW_ACTION_TYPE_QUEUE,
 252         RTE_FLOW_ACTION_TYPE_RSS,
 253         RTE_FLOW_ACTION_TYPE_VOID,
 254         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 255         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 256         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 257 };
 258
 259 /* Egress actions */
 260 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 261         RTE_FLOW_ACTION_TYPE_COUNT,
 262         RTE_FLOW_ACTION_TYPE_DROP,
 263         RTE_FLOW_ACTION_TYPE_JUMP,
 264         RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
 265         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
 266         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
 267         RTE_FLOW_ACTION_TYPE_PORT_ID,
 268         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 269         RTE_FLOW_ACTION_TYPE_VOID,
 270         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 271         RTE_FLOW_ACTION_TYPE_END,
 272 };
 273
 274 static const struct enic_fm_items enic_fm_items[] = {
 275         [RTE_FLOW_ITEM_TYPE_RAW] = {
 276                 .copy_item = enic_fm_copy_item_raw,
 277                 .valid_start_item = 0,
 278                 .prev_items = (const enum rte_flow_item_type[]) {
 279                                RTE_FLOW_ITEM_TYPE_UDP,
 280                                RTE_FLOW_ITEM_TYPE_END,
 281                 },
 282         },
 283         [RTE_FLOW_ITEM_TYPE_ETH] = {
 284                 .copy_item = enic_fm_copy_item_eth,
 285                 .valid_start_item = 1,
 286                 .prev_items = (const enum rte_flow_item_type[]) {
 287                                RTE_FLOW_ITEM_TYPE_END,
 288                 },
 289         },
 290         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 291                 .copy_item = enic_fm_copy_item_vlan,
 292                 .valid_start_item = 1,
 293                 .prev_items = (const enum rte_flow_item_type[]) {
 294                                RTE_FLOW_ITEM_TYPE_ETH,
 295                                RTE_FLOW_ITEM_TYPE_END,
 296                 },
 297         },
 298         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 299                 .copy_item = enic_fm_copy_item_ipv4,
 300                 .valid_start_item = 1,
 301                 .prev_items = (const enum rte_flow_item_type[]) {
 302                                RTE_FLOW_ITEM_TYPE_ETH,
 303                                RTE_FLOW_ITEM_TYPE_VLAN,
 304                                RTE_FLOW_ITEM_TYPE_END,
 305                 },
 306         },
 307         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 308                 .copy_item = enic_fm_copy_item_ipv6,
 309                 .valid_start_item = 1,
 310                 .prev_items = (const enum rte_flow_item_type[]) {
 311                                RTE_FLOW_ITEM_TYPE_ETH,
 312                                RTE_FLOW_ITEM_TYPE_VLAN,
 313                                RTE_FLOW_ITEM_TYPE_END,
 314                 },
 315         },
 316         [RTE_FLOW_ITEM_TYPE_UDP] = {
 317                 .copy_item = enic_fm_copy_item_udp,
 318                 .valid_start_item = 1,
 319                 .prev_items = (const enum rte_flow_item_type[]) {
 320                                RTE_FLOW_ITEM_TYPE_IPV4,
 321                                RTE_FLOW_ITEM_TYPE_IPV6,
 322                                RTE_FLOW_ITEM_TYPE_END,
 323                 },
 324         },
 325         [RTE_FLOW_ITEM_TYPE_TCP] = {
 326                 .copy_item = enic_fm_copy_item_tcp,
 327                 .valid_start_item = 1,
 328                 .prev_items = (const enum rte_flow_item_type[]) {
 329                                RTE_FLOW_ITEM_TYPE_IPV4,
 330                                RTE_FLOW_ITEM_TYPE_IPV6,
 331                                RTE_FLOW_ITEM_TYPE_END,
 332                 },
 333         },
 334         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 335                 .copy_item = enic_fm_copy_item_sctp,
 336                 .valid_start_item = 0,
 337                 .prev_items = (const enum rte_flow_item_type[]) {
 338                                RTE_FLOW_ITEM_TYPE_IPV4,
 339                                RTE_FLOW_ITEM_TYPE_IPV6,
 340                                RTE_FLOW_ITEM_TYPE_END,
 341                 },
 342         },
 343         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 344                 .copy_item = enic_fm_copy_item_vxlan,
 345                 .valid_start_item = 1,
 346                 .prev_items = (const enum rte_flow_item_type[]) {
 347                                RTE_FLOW_ITEM_TYPE_UDP,
 348                                RTE_FLOW_ITEM_TYPE_END,
 349                 },
 350         },
 351         [RTE_FLOW_ITEM_TYPE_GTP] = {
 352                 .copy_item = enic_fm_copy_item_gtp,
 353                 .valid_start_item = 0,
 354                 .prev_items = (const enum rte_flow_item_type[]) {
 355                                RTE_FLOW_ITEM_TYPE_UDP,
 356                                RTE_FLOW_ITEM_TYPE_END,
 357                 },
 358         },
 359         [RTE_FLOW_ITEM_TYPE_GTPC] = {
 360                 .copy_item = enic_fm_copy_item_gtp,
 361                 .valid_start_item = 1,
 362                 .prev_items = (const enum rte_flow_item_type[]) {
 363                                RTE_FLOW_ITEM_TYPE_UDP,
 364                                RTE_FLOW_ITEM_TYPE_END,
 365                 },
 366         },
 367         [RTE_FLOW_ITEM_TYPE_GTPU] = {
 368                 .copy_item = enic_fm_copy_item_gtp,
 369                 .valid_start_item = 1,
 370                 .prev_items = (const enum rte_flow_item_type[]) {
 371                                RTE_FLOW_ITEM_TYPE_UDP,
 372                                RTE_FLOW_ITEM_TYPE_END,
 373                 },
 374         },
 375         [RTE_FLOW_ITEM_TYPE_GENEVE] = {
 376                 .copy_item = enic_fm_copy_item_geneve,
 377                 .valid_start_item = 1,
 378                 .prev_items = (const enum rte_flow_item_type[]) {
 379                                RTE_FLOW_ITEM_TYPE_ETH,
 380                                RTE_FLOW_ITEM_TYPE_IPV4,
 381                                RTE_FLOW_ITEM_TYPE_IPV6,
 382                                RTE_FLOW_ITEM_TYPE_UDP,
 383                                RTE_FLOW_ITEM_TYPE_END,
 384                 },
 385         },
 386         [RTE_FLOW_ITEM_TYPE_GENEVE_OPT] = {
 387                 .copy_item = enic_fm_copy_item_geneve_opt,
 388                 .valid_start_item = 1,
 389                 /* Can match at most 1 option */
 390                 .prev_items = (const enum rte_flow_item_type[]) {
 391                                RTE_FLOW_ITEM_TYPE_GENEVE,
 392                                RTE_FLOW_ITEM_TYPE_END,
 393                 },
 394         },
 395 };
 396
 397 static int
 398 enic_fm_copy_item_eth(struct copy_item_args *arg)
 399 {
 400         const struct rte_flow_item *item = arg->item;
 401         const struct rte_flow_item_eth *spec = item->spec;
 402         const struct rte_flow_item_eth *mask = item->mask;
 403         const uint8_t lvl = arg->header_level;
 404         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 405         struct fm_header_set *fm_data, *fm_mask;
 406
 407         ENICPMD_FUNC_TRACE();
 408         /* Match all if no spec */
 409         if (!spec)
 410                 return 0;
 411         if (!mask)
 412                 mask = &rte_flow_item_eth_mask;
 413         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 414         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 415         fm_data->fk_header_select |= FKH_ETHER;
 416         fm_mask->fk_header_select |= FKH_ETHER;
 417         memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
 418         memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
 419         return 0;
 420 }
 421
 422 static int
 423 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 424 {
 425         const struct rte_flow_item *item = arg->item;
 426         const struct rte_flow_item_vlan *spec = item->spec;
 427         const struct rte_flow_item_vlan *mask = item->mask;
 428         const uint8_t lvl = arg->header_level;
 429         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 430         struct fm_header_set *fm_data, *fm_mask;
 431         struct rte_ether_hdr *eth_mask;
 432         struct rte_ether_hdr *eth_val;
 433         uint32_t meta;
 434
 435         ENICPMD_FUNC_TRACE();
 436         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 437         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 438         /* Outer and inner packet vlans need different flags */
 439         meta = FKM_VLAN_PRES;
 440         if (lvl > 0)
 441                 meta = FKM_QTAG;
 442         fm_data->fk_metadata |= meta;
 443         fm_mask->fk_metadata |= meta;
 444
 445         /* Match all if no spec */
 446         if (!spec)
 447                 return 0;
 448         if (!mask)
 449                 mask = &rte_flow_item_vlan_mask;
 450
 451         eth_mask = (void *)&fm_mask->l2.eth;
 452         eth_val = (void *)&fm_data->l2.eth;
 453
 454         /*
 455          * Outer TPID cannot be matched. If inner_type is 0, use what is
 456          * in the eth header.
 457          */
 458         if (eth_mask->ether_type && mask->inner_type)
 459                 return -ENOTSUP;
 460
 461         /*
 462          * When packet matching, the VIC always compares vlan-stripped
 463          * L2, regardless of vlan stripping settings. So, the inner type
 464          * from vlan becomes the ether type of the eth header.
 465          */
 466         if (mask->inner_type) {
 467                 eth_mask->ether_type = mask->inner_type;
 468                 eth_val->ether_type = spec->inner_type;
 469         }
 470         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 471         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 472         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 473         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 474         return 0;
 475 }
 476
 477 static int
 478 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 479 {
 480         const struct rte_flow_item *item = arg->item;
 481         const struct rte_flow_item_ipv4 *spec = item->spec;
 482         const struct rte_flow_item_ipv4 *mask = item->mask;
 483         const uint8_t lvl = arg->header_level;
 484         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 485         struct fm_header_set *fm_data, *fm_mask;
 486
 487         ENICPMD_FUNC_TRACE();
 488         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 489         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 490         fm_data->fk_metadata |= FKM_IPV4;
 491         fm_mask->fk_metadata |= FKM_IPV4;
 492
 493         if (!spec)
 494                 return 0;
 495         if (!mask)
 496                 mask = &rte_flow_item_ipv4_mask;
 497
 498         fm_data->fk_header_select |= FKH_IPV4;
 499         fm_mask->fk_header_select |= FKH_IPV4;
 500         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 501         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 502         return 0;
 503 }
 504
 505 static int
 506 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 507 {
 508         const struct rte_flow_item *item = arg->item;
 509         const struct rte_flow_item_ipv6 *spec = item->spec;
 510         const struct rte_flow_item_ipv6 *mask = item->mask;
 511         const uint8_t lvl = arg->header_level;
 512         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 513         struct fm_header_set *fm_data, *fm_mask;
 514
 515         ENICPMD_FUNC_TRACE();
 516         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 517         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 518         fm_data->fk_metadata |= FKM_IPV6;
 519         fm_mask->fk_metadata |= FKM_IPV6;
 520
 521         if (!spec)
 522                 return 0;
 523         if (!mask)
 524                 mask = &rte_flow_item_ipv6_mask;
 525
 526         fm_data->fk_header_select |= FKH_IPV6;
 527         fm_mask->fk_header_select |= FKH_IPV6;
 528         memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
 529         memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
 530         return 0;
 531 }
 532
 533 static int
 534 enic_fm_copy_item_udp(struct copy_item_args *arg)
 535 {
 536         const struct rte_flow_item *item = arg->item;
 537         const struct rte_flow_item_udp *spec = item->spec;
 538         const struct rte_flow_item_udp *mask = item->mask;
 539         const uint8_t lvl = arg->header_level;
 540         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 541         struct fm_header_set *fm_data, *fm_mask;
 542
 543         ENICPMD_FUNC_TRACE();
 544         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 545         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 546         fm_data->fk_metadata |= FKM_UDP;
 547         fm_mask->fk_metadata |= FKM_UDP;
 548
 549         if (!spec)
 550                 return 0;
 551         if (!mask)
 552                 mask = &rte_flow_item_udp_mask;
 553
 554         fm_data->fk_header_select |= FKH_UDP;
 555         fm_mask->fk_header_select |= FKH_UDP;
 556         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 557         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 558         return 0;
 559 }
 560
 561 static int
 562 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 563 {
 564         const struct rte_flow_item *item = arg->item;
 565         const struct rte_flow_item_tcp *spec = item->spec;
 566         const struct rte_flow_item_tcp *mask = item->mask;
 567         const uint8_t lvl = arg->header_level;
 568         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 569         struct fm_header_set *fm_data, *fm_mask;
 570
 571         ENICPMD_FUNC_TRACE();
 572         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 573         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 574         fm_data->fk_metadata |= FKM_TCP;
 575         fm_mask->fk_metadata |= FKM_TCP;
 576
 577         if (!spec)
 578                 return 0;
 579         if (!mask)
 580                 mask = &rte_flow_item_tcp_mask;
 581
 582         fm_data->fk_header_select |= FKH_TCP;
 583         fm_mask->fk_header_select |= FKH_TCP;
 584         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 585         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 586         return 0;
 587 }
 588
 589 static int
 590 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 591 {
 592         const struct rte_flow_item *item = arg->item;
 593         const struct rte_flow_item_sctp *spec = item->spec;
 594         const struct rte_flow_item_sctp *mask = item->mask;
 595         const uint8_t lvl = arg->header_level;
 596         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 597         struct fm_header_set *fm_data, *fm_mask;
 598         uint8_t *ip_proto_mask = NULL;
 599         uint8_t *ip_proto = NULL;
 600         uint32_t l3_fkh;
 601
 602         ENICPMD_FUNC_TRACE();
 603         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 604         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 605         /*
 606          * The NIC filter API has no flags for "match sctp", so explicitly
 607          * set the protocol number in the IP pattern.
 608          */
 609         if (fm_data->fk_metadata & FKM_IPV4) {
 610                 struct rte_ipv4_hdr *ip;
 611                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 612                 ip_proto_mask = &ip->next_proto_id;
 613                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 614                 ip_proto = &ip->next_proto_id;
 615                 l3_fkh = FKH_IPV4;
 616         } else if (fm_data->fk_metadata & FKM_IPV6) {
 617                 struct rte_ipv6_hdr *ip;
 618                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 619                 ip_proto_mask = &ip->proto;
 620                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 621                 ip_proto = &ip->proto;
 622                 l3_fkh = FKH_IPV6;
 623         } else {
 624                 /* Need IPv4/IPv6 pattern first */
 625                 return -EINVAL;
 626         }
 627         *ip_proto = IPPROTO_SCTP;
 628         *ip_proto_mask = 0xff;
 629         fm_data->fk_header_select |= l3_fkh;
 630         fm_mask->fk_header_select |= l3_fkh;
 631
 632         if (!spec)
 633                 return 0;
 634         if (!mask)
 635                 mask = &rte_flow_item_sctp_mask;
 636
 637         fm_data->fk_header_select |= FKH_L4RAW;
 638         fm_mask->fk_header_select |= FKH_L4RAW;
 639         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 640         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 641         return 0;
 642 }
 643
 644 static int
 645 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 646 {
 647         const struct rte_flow_item *item = arg->item;
 648         const struct rte_flow_item_vxlan *spec = item->spec;
 649         const struct rte_flow_item_vxlan *mask = item->mask;
 650         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 651         struct fm_header_set *fm_data, *fm_mask;
 652
 653         ENICPMD_FUNC_TRACE();
 654         /* Only 2 header levels (outer and inner) allowed */
 655         if (arg->header_level > 0)
 656                 return -EINVAL;
 657
 658         fm_data = &entry->ftm_data.fk_hdrset[0];
 659         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 660         fm_data->fk_metadata |= FKM_VXLAN;
 661         fm_mask->fk_metadata |= FKM_VXLAN;
 662         /* items from here on out are inner header items */
 663         arg->header_level = 1;
 664
 665         /* Match all if no spec */
 666         if (!spec)
 667                 return 0;
 668         if (!mask)
 669                 mask = &rte_flow_item_vxlan_mask;
 670
 671         fm_data->fk_header_select |= FKH_VXLAN;
 672         fm_mask->fk_header_select |= FKH_VXLAN;
 673         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 674         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 675         return 0;
 676 }
 677
 678 static int
 679 enic_fm_copy_item_gtp(struct copy_item_args *arg)
 680 {
 681         const struct rte_flow_item *item = arg->item;
 682         const struct rte_flow_item_gtp *spec = item->spec;
 683         const struct rte_flow_item_gtp *mask = item->mask;
 684         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 685         struct fm_header_set *fm_data, *fm_mask;
 686         int off;
 687         uint16_t udp_gtp_uc_port_be = 0;
 688
 689         ENICPMD_FUNC_TRACE();
 690         /* Only 2 header levels (outer and inner) allowed */
 691         if (arg->header_level > 0)
 692                 return -EINVAL;
 693
 694         fm_data = &entry->ftm_data.fk_hdrset[0];
 695         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 696
 697         switch (item->type) {
 698         case RTE_FLOW_ITEM_TYPE_GTP:
 699         {
 700                 /* For vanilla GTP, the UDP destination port must be specified
 701                  * but value of the port is not enforced here.
 702                  */
 703                 if (!(fm_data->fk_metadata & FKM_UDP) ||
 704                     !(fm_data->fk_header_select & FKH_UDP) ||
 705                     fm_data->l4.udp.fk_dest == 0)
 706                         return -EINVAL;
 707                 if (!(fm_mask->fk_metadata & FKM_UDP) ||
 708                     !(fm_mask->fk_header_select & FKH_UDP) ||
 709                     fm_mask->l4.udp.fk_dest != 0xFFFF)
 710                         return -EINVAL;
 711                 break;
 712         }
 713         case RTE_FLOW_ITEM_TYPE_GTPC:
 714         {
 715                 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPC_UDP_PORT);
 716                 break;
 717         }
 718         case RTE_FLOW_ITEM_TYPE_GTPU:
 719         {
 720                 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPU_UDP_PORT);
 721                 break;
 722         }
 723         default:
 724                 RTE_ASSERT(0);
 725         }
 726
 727         /* The GTP-C or GTP-U UDP destination port must be matched. */
 728         if (udp_gtp_uc_port_be) {
 729                 if (fm_data->fk_metadata & FKM_UDP &&
 730                     fm_data->fk_header_select & FKH_UDP &&
 731                     fm_data->l4.udp.fk_dest != udp_gtp_uc_port_be)
 732                         return -EINVAL;
 733                 if (fm_mask->fk_metadata & FKM_UDP &&
 734                     fm_mask->fk_header_select & FKH_UDP &&
 735                     fm_mask->l4.udp.fk_dest != 0xFFFF)
 736                         return -EINVAL;
 737
 738                 /* In any case, add match for GTP-C GTP-U UDP dst port */
 739                 fm_data->fk_metadata |= FKM_UDP;
 740                 fm_data->fk_header_select |= FKH_UDP;
 741                 fm_data->l4.udp.fk_dest = udp_gtp_uc_port_be;
 742                 fm_mask->fk_metadata |= FKM_UDP;
 743                 fm_mask->fk_header_select |= FKH_UDP;
 744                 fm_mask->l4.udp.fk_dest = 0xFFFF;
 745         }
 746
 747         /* NIC does not support GTP tunnels. No Items are allowed after this.
 748          * This prevents the specification of further items.
 749          */
 750         arg->header_level = 0;
 751
 752         /* Match all if no spec */
 753         if (!spec)
 754                 return 0;
 755         if (!mask)
 756                 mask = &rte_flow_item_gtp_mask;
 757
 758         /*
 759          * Use the raw L4 buffer to match GTP as fm_header_set does not have
 760          * GTP header. UDP dst port must be specific. Using the raw buffer
 761          * does not affect such UDP item, since we skip UDP in the raw buffer.
 762          */
 763         fm_data->fk_header_select |= FKH_L4RAW;
 764         fm_mask->fk_header_select |= FKH_L4RAW;
 765         off = sizeof(fm_data->l4.udp);
 766         memcpy(&fm_data->l4.rawdata[off], spec, sizeof(*spec));
 767         memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(*mask));
 768         return 0;
 769 }
 770
 771 static int
 772 enic_fm_copy_item_geneve(struct copy_item_args *arg)
 773 {
 774         const struct rte_flow_item *item = arg->item;
 775         const struct rte_flow_item_geneve *spec = item->spec;
 776         const struct rte_flow_item_geneve *mask = item->mask;
 777         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 778         struct fm_header_set *fm_data, *fm_mask;
 779         int off;
 780
 781         ENICPMD_FUNC_TRACE();
 782         /* Only 2 header levels (outer and inner) allowed */
 783         if (arg->header_level > 0)
 784                 return -EINVAL;
 785
 786         fm_data = &entry->ftm_data.fk_hdrset[0];
 787         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 788         fm_data->fk_metadata |= FKM_GENEVE;
 789         fm_mask->fk_metadata |= FKM_GENEVE;
 790         /* items from here on out are inner header items, except options */
 791         arg->header_level = 1;
 792
 793         /* Match all if no spec */
 794         if (!spec)
 795                 return 0;
 796         if (!mask)
 797                 mask = &rte_flow_item_geneve_mask;
 798
 799         /*
 800          * Use the raw L4 buffer to match geneve as fm_header_set does
 801          * not have geneve header. A UDP item may precede the geneve
 802          * item. Using the raw buffer does not affect such UDP item,
 803          * since we skip UDP in the raw buffer.
 804          */
 805         fm_data->fk_header_select |= FKH_L4RAW;
 806         fm_mask->fk_header_select |= FKH_L4RAW;
 807         off = sizeof(fm_data->l4.udp);
 808         memcpy(&fm_data->l4.rawdata[off], spec, sizeof(struct rte_geneve_hdr));
 809         memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(struct rte_geneve_hdr));
 810         return 0;
 811 }
 812
 813 static int
 814 enic_fm_copy_item_geneve_opt(struct copy_item_args *arg)
 815 {
 816         const struct rte_flow_item *item = arg->item;
 817         const struct rte_flow_item_geneve_opt *spec = item->spec;
 818         const struct rte_flow_item_geneve_opt *mask = item->mask;
 819         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 820         struct fm_header_set *fm_data, *fm_mask;
 821         struct rte_geneve_hdr *geneve;
 822         int off, len;
 823
 824         ENICPMD_FUNC_TRACE();
 825         fm_data = &entry->ftm_data.fk_hdrset[0];
 826         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 827         /* Match all if no spec */
 828         if (!spec)
 829                 return 0;
 830         if (!mask)
 831                 mask = &rte_flow_item_geneve_opt_mask;
 832
 833         if (spec->option_len > 0 &&
 834             (spec->data == NULL || mask->data == NULL)) {
 835                 return rte_flow_error_set(arg->error, EINVAL,
 836                         RTE_FLOW_ERROR_TYPE_ITEM,
 837                         NULL, "enic: geneve_opt unexpected null data");
 838         }
 839         /*
 840          * Geneve item must already be in the raw buffer. Append the
 841          * option pattern to it. There are two limitations.
 842          * (1) Can match only the 1st option, the first one following Geneve
 843          * (2) Geneve header must specify option length, as HW does not
 844          *     have "has Geneve option" flag.
 845          */
 846         RTE_ASSERT((fm_data->fk_header_select & FKH_L4RAW) != 0);
 847         RTE_ASSERT((fm_mask->fk_header_select & FKH_L4RAW) != 0);
 848         off = sizeof(fm_data->l4.udp);
 849         geneve = (struct rte_geneve_hdr *)&fm_data->l4.rawdata[off];
 850         if (geneve->opt_len == 0) {
 851                 return rte_flow_error_set(arg->error, EINVAL,
 852                         RTE_FLOW_ERROR_TYPE_ITEM,
 853                         NULL, "enic: geneve_opt requires non-zero geneve option length");
 854         }
 855         geneve = (struct rte_geneve_hdr *)&fm_mask->l4.rawdata[off];
 856         if (geneve->opt_len == 0) {
 857                 return rte_flow_error_set(arg->error, EINVAL,
 858                         RTE_FLOW_ERROR_TYPE_ITEM,
 859                         NULL, "enic: geneve_opt requires non-zero geneve option length mask");
 860         }
 861         off = sizeof(fm_data->l4.udp) + sizeof(struct rte_geneve_hdr);
 862         if (off + (spec->option_len + 1) * 4 > FM_LAYER_SIZE) {
 863                 return rte_flow_error_set(arg->error, EINVAL,
 864                         RTE_FLOW_ERROR_TYPE_ITEM,
 865                         NULL, "enic: geneve_opt too large");
 866         }
 867         /* Copy option header */
 868         memcpy(&fm_data->l4.rawdata[off], spec, 4);
 869         memcpy(&fm_mask->l4.rawdata[off], mask, 4);
 870         /* Copy option data */
 871         if (spec->option_len > 0) {
 872                 off += 4;
 873                 len = spec->option_len * 4;
 874                 memcpy(&fm_data->l4.rawdata[off], spec->data, len);
 875                 memcpy(&fm_mask->l4.rawdata[off], mask->data, len);
 876         }
 877         return 0;
 878 }
 879
 880 /*
 881  * Currently, raw pattern match is very limited. It is intended for matching
 882  * UDP tunnel header (e.g. vxlan or geneve).
 883  */
 884 static int
 885 enic_fm_copy_item_raw(struct copy_item_args *arg)
 886 {
 887         const struct rte_flow_item *item = arg->item;
 888         const struct rte_flow_item_raw *spec = item->spec;
 889         const struct rte_flow_item_raw *mask = item->mask;
 890         const uint8_t lvl = arg->header_level;
 891         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 892         struct fm_header_set *fm_data, *fm_mask;
 893
 894         ENICPMD_FUNC_TRACE();
 895         /* Cannot be used for inner packet */
 896         if (lvl > 0)
 897                 return -EINVAL;
 898         /* Need both spec and mask */
 899         if (!spec || !mask)
 900                 return -EINVAL;
 901         /* Only supports relative with offset 0 */
 902         if (!spec->relative || spec->offset != 0 || spec->search ||
 903             spec->limit)
 904                 return -EINVAL;
 905         /* Need non-null pattern that fits within the NIC's filter pattern */
 906         if (spec->length == 0 ||
 907             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 908             !spec->pattern || !mask->pattern)
 909                 return -EINVAL;
 910         /*
 911          * Mask fields, including length, are often set to zero. Assume that
 912          * means "same as spec" to avoid breaking existing apps. If length
 913          * is not zero, then it should be >= spec length.
 914          *
 915          * No more pattern follows this, so append to the L4 layer instead of
 916          * L5 to work with both recent and older VICs.
 917          */
 918         if (mask->length != 0 && mask->length < spec->length)
 919                 return -EINVAL;
 920
 921         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 922         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 923         fm_data->fk_header_select |= FKH_L4RAW;
 924         fm_mask->fk_header_select |= FKH_L4RAW;
 925         fm_data->fk_header_select &= ~FKH_UDP;
 926         fm_mask->fk_header_select &= ~FKH_UDP;
 927         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 928                spec->pattern, spec->length);
 929         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 930                mask->pattern, spec->length);
 931         return 0;
 932 }
 933
 934 static int
 935 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
 936 {
 937         return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
 938 }
 939
 940 static int
 941 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 942                struct fm_key_template *key, int entries,
 943                struct enic_fm_fet **fet_out)
 944 {
 945         struct fm_exact_match_table *cmd;
 946         struct fm_header_set *hdr;
 947         struct enic_fm_fet *fet;
 948         uint64_t args[3];
 949         int ret;
 950
 951         ENICPMD_FUNC_TRACE();
 952         fet = calloc(1, sizeof(struct enic_fm_fet));
 953         if (fet == NULL)
 954                 return -ENOMEM;
 955         cmd = &fm->cmd.va->fm_exact_match_table;
 956         memset(cmd, 0, sizeof(*cmd));
 957         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 958         cmd->fet_stage = FM_STAGE_LAST;
 959         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 960         if (key == NULL) {
 961                 hdr = &cmd->fet_key.fk_hdrset[0];
 962                 memset(hdr, 0, sizeof(*hdr));
 963                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 964                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 965                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 966                 hdr->l4.udp.fk_source = 0xFFFF;
 967                 hdr->l4.udp.fk_dest = 0xFFFF;
 968                 fet->default_key = 1;
 969         } else {
 970                 memcpy(&cmd->fet_key, key, sizeof(*key));
 971                 memcpy(&fet->key, key, sizeof(*key));
 972                 fet->default_key = 0;
 973         }
 974         cmd->fet_key.fk_packet_tag = 1;
 975
 976         args[0] = FM_EXACT_TABLE_ALLOC;
 977         args[1] = fm->cmd.pa;
 978         ret = flowman_cmd(fm, args, 2);
 979         if (ret) {
 980                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 981                 free(fet);
 982                 return ret;
 983         }
 984         fet->handle = args[0];
 985         fet->ingress = ingress;
 986         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 987                     fet->handle);
 988         *fet_out = fet;
 989         return 0;
 990 }
 991
 992 static void
 993 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 994 {
 995         ENICPMD_FUNC_TRACE();
 996         enic_fm_tbl_free(fm, fet->handle);
 997         if (!fet->default_key)
 998                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 999         free(fet);
1000 }
1001
1002 /*
1003  * Get the exact match table for the given combination of
1004  * <group, ingress, key>. Allocate one on the fly as necessary.
1005  */
1006 static int
1007 enic_fet_get(struct enic_flowman *fm,
1008              uint32_t group,
1009              uint8_t ingress,
1010              struct fm_key_template *key,
1011              struct enic_fm_fet **fet_out,
1012              struct rte_flow_error *error)
1013 {
1014         struct enic_fm_fet *fet;
1015
1016         ENICPMD_FUNC_TRACE();
1017         /* See if we already have this table open */
1018         TAILQ_FOREACH(fet, &fm->fet_list, list) {
1019                 if (fet->group == group && fet->ingress == ingress)
1020                         break;
1021         }
1022         if (fet == NULL) {
1023                 /* Jumping to a non-existing group? Use the default table */
1024                 if (key == NULL) {
1025                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
1026                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
1027                         return rte_flow_error_set(error, EINVAL,
1028                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1029                                 NULL, "enic: cannot get exact match table");
1030                 }
1031                 fet->group = group;
1032                 /* Default table is never on the open table list */
1033                 if (!fet->default_key)
1034                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
1035         }
1036         fet->ref++;
1037         *fet_out = fet;
1038         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
1039                     fet->default_key ? "default" : "",
1040                     fet->ingress ? "ingress" : "egress",
1041                     fet->group, fet->ref);
1042         return 0;
1043 }
1044
1045 static void
1046 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
1047 {
1048         ENICPMD_FUNC_TRACE();
1049         RTE_ASSERT(fet->ref > 0);
1050         fet->ref--;
1051         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
1052                     fet->default_key ? "default" : "",
1053                     fet->ingress ? "ingress" : "egress",
1054                     fet->group, fet->ref);
1055         if (fet->ref == 0)
1056                 enic_fet_free(fm, fet);
1057 }
1058
1059 /* Return 1 if current item is valid on top of the previous one. */
1060 static int
1061 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
1062                        const struct enic_fm_items *item_info,
1063                        uint8_t is_first_item)
1064 {
1065         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
1066
1067         ENICPMD_FUNC_TRACE();
1068         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
1069                 if (prev_item == *allowed_items)
1070                         return 1;
1071         }
1072
1073         /* This is the first item in the stack. Check if that's cool */
1074         if (is_first_item && item_info->valid_start_item)
1075                 return 1;
1076         return 0;
1077 }
1078
1079 /*
1080  * Build the flow manager match entry structure from the provided pattern.
1081  * The pattern is validated as the items are copied.
1082  */
1083 static int
1084 enic_fm_copy_entry(struct enic_flowman *fm,
1085                    const struct rte_flow_item pattern[],
1086                    struct rte_flow_error *error)
1087 {
1088         const struct enic_fm_items *item_info;
1089         enum rte_flow_item_type prev_item;
1090         const struct rte_flow_item *item;
1091         struct copy_item_args args;
1092         uint8_t prev_header_level;
1093         uint8_t is_first_item;
1094         int ret;
1095
1096         ENICPMD_FUNC_TRACE();
1097         item = pattern;
1098         is_first_item = 1;
1099         prev_item = RTE_FLOW_ITEM_TYPE_END;
1100
1101         args.fm_tcam_entry = &fm->tcam_entry;
1102         args.header_level = 0;
1103         prev_header_level = 0;
1104         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
1105                 /*
1106                  * Get info about how to validate and copy the item. If NULL
1107                  * is returned the nic does not support the item.
1108                  */
1109                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
1110                         continue;
1111
1112                 item_info = &enic_fm_items[item->type];
1113
1114                 if (item->type >= RTE_DIM(enic_fm_items) ||
1115                     item_info->copy_item == NULL) {
1116                         return rte_flow_error_set(error, ENOTSUP,
1117                                 RTE_FLOW_ERROR_TYPE_ITEM,
1118                                 NULL, "enic: unsupported item");
1119                 }
1120                 /*
1121                  * Check vNIC feature dependencies. Geneve item needs
1122                  * Geneve offload feature
1123                  */
1124                 if (item->type == RTE_FLOW_ITEM_TYPE_GENEVE &&
1125                     !fm->user_enic->geneve) {
1126                         return rte_flow_error_set(error, ENOTSUP,
1127                                 RTE_FLOW_ERROR_TYPE_ITEM,
1128                                 NULL, "enic: geneve not supported");
1129                 }
1130                 /* check to see if item stacking is valid */
1131                 if (!fm_item_stacking_valid(prev_item, item_info,
1132                                             is_first_item))
1133                         goto stacking_error;
1134
1135                 args.item = item;
1136                 args.error = error;
1137                 if (error)
1138                         error->type = RTE_FLOW_ERROR_TYPE_NONE;
1139                 ret = item_info->copy_item(&args);
1140                 if (ret) {
1141                         /* If copy_item set the error, return that */
1142                         if (error->type != RTE_FLOW_ERROR_TYPE_NONE)
1143                                 return ret;
1144                         goto item_not_supported;
1145                 }
1146                 /* Going from outer to inner? Treat it as a new packet start */
1147                 if (prev_header_level != args.header_level) {
1148                         prev_item = RTE_FLOW_ITEM_TYPE_END;
1149                         is_first_item = 1;
1150                 } else {
1151                         prev_item = item->type;
1152                         is_first_item = 0;
1153                 }
1154                 prev_header_level = args.header_level;
1155         }
1156         return 0;
1157
1158 item_not_supported:
1159         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
1160                                   NULL, "enic: unsupported item type");
1161
1162 stacking_error:
1163         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
1164                                   item, "enic: unsupported item stack");
1165 }
1166
1167 static void
1168 flow_item_skip_void(const struct rte_flow_item **item)
1169 {
1170         for ( ; ; (*item)++)
1171                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
1172                         return;
1173 }
1174
1175 static void
1176 append_template(void **template, uint8_t *off, const void *data, int len)
1177 {
1178         memcpy(*template, data, len);
1179         *template = (char *)*template + len;
1180         *off = *off + len;
1181 }
1182
1183 static int
1184 enic_fm_append_action_op(struct enic_flowman *fm,
1185                          struct fm_action_op *fm_op,
1186                          struct rte_flow_error *error)
1187 {
1188         int count;
1189
1190         count = fm->action_op_count;
1191         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
1192                     count, fm_op->fa_op);
1193         if (count == FM_ACTION_OP_MAX) {
1194                 return rte_flow_error_set(error, EINVAL,
1195                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1196                         "too many action operations");
1197         }
1198         fm->action.fma_action_ops[count] = *fm_op;
1199         fm->action_op_count = count + 1;
1200         return 0;
1201 }
1202
1203 static struct fm_action_op *
1204 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
1205 {
1206         struct fm_action_op *op;
1207         int i;
1208
1209         for (i = 0; i < fm->action_op_count; i++) {
1210                 op = &fm->action.fma_action_ops[i];
1211                 if (op->fa_op == opcode)
1212                         return op;
1213         }
1214         return NULL;
1215 }
1216
1217 /* NIC requires that 1st steer appear before decap.
1218  * Correct example: steer, decap, steer, steer, ...
1219  */
1220 static void
1221 enic_fm_reorder_action_op(struct enic_flowman *fm)
1222 {
1223         struct fm_action_op *op, *steer, *decap;
1224         struct fm_action_op tmp_op;
1225
1226         ENICPMD_FUNC_TRACE();
1227         /* Find 1st steer and decap */
1228         op = fm->action.fma_action_ops;
1229         steer = NULL;
1230         decap = NULL;
1231         while (op->fa_op != FMOP_END) {
1232                 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
1233                                op->fa_op == FMOP_DECAP_STRIP))
1234                         decap = op;
1235                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
1236                         steer = op;
1237                 op++;
1238         }
1239         /* If decap is before steer, swap */
1240         if (steer && decap && decap < steer) {
1241                 op = fm->action.fma_action_ops;
1242                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
1243                             (long)(decap - op), (long)(steer - op));
1244                 tmp_op = *decap;
1245                 *decap = *steer;
1246                 *steer = tmp_op;
1247         }
1248 }
1249
1250 /* VXLAN decap is done via flowman compound action */
1251 static int
1252 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
1253                          struct fm_tcam_match_entry *fmt,
1254                          const struct rte_flow_action *action,
1255                          struct rte_flow_error *error)
1256 {
1257         struct fm_header_set *fm_data;
1258         struct fm_action_op fm_op;
1259
1260         ENICPMD_FUNC_TRACE();
1261         fm_data = &fmt->ftm_data.fk_hdrset[0];
1262         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
1263                 return rte_flow_error_set(error, EINVAL,
1264                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1265                         "vxlan-decap: vxlan must be in pattern");
1266         }
1267
1268         memset(&fm_op, 0, sizeof(fm_op));
1269         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
1270         return enic_fm_append_action_op(fm, &fm_op, error);
1271 }
1272
1273 /* Generate a reasonable source port number */
1274 static uint16_t
1275 gen_src_port(void)
1276 {
1277         /* Min/max below are the default values in OVS-DPDK and Linux */
1278         uint16_t p = rte_rand();
1279         p = RTE_MAX(p, 32768);
1280         p = RTE_MIN(p, 61000);
1281         return rte_cpu_to_be_16(p);
1282 }
1283
1284 /* VXLAN encap is done via flowman compound action */
1285 static int
1286 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1287                          const struct rte_flow_item *item,
1288                          struct rte_flow_error *error)
1289 {
1290         struct fm_action_op fm_op;
1291         struct rte_ether_hdr *eth;
1292         struct rte_udp_hdr *udp;
1293         uint16_t *ethertype;
1294         void *template;
1295         uint8_t off;
1296
1297         ENICPMD_FUNC_TRACE();
1298         memset(&fm_op, 0, sizeof(fm_op));
1299         fm_op.fa_op = FMOP_ENCAP;
1300         template = fm->action.fma_data;
1301         off = 0;
1302         /*
1303          * Copy flow items to the flowman template starting L2.
1304          * L2 must be ethernet.
1305          */
1306         flow_item_skip_void(&item);
1307         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1308                 return rte_flow_error_set(error, EINVAL,
1309                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1310                         "vxlan-encap: first item should be ethernet");
1311         eth = (struct rte_ether_hdr *)template;
1312         ethertype = &eth->ether_type;
1313         append_template(&template, &off, item->spec,
1314                         sizeof(struct rte_ether_hdr));
1315         item++;
1316         flow_item_skip_void(&item);
1317         /* Optional VLAN */
1318         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1319                 const struct rte_flow_item_vlan *spec;
1320
1321                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1322                 spec = item->spec;
1323                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1324                 item++;
1325                 flow_item_skip_void(&item);
1326         }
1327         /* L3 must be IPv4, IPv6 */
1328         switch (item->type) {
1329         case RTE_FLOW_ITEM_TYPE_IPV4:
1330         {
1331                 struct rte_ipv4_hdr *ip4;
1332
1333                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1334                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1335                 ip4 = (struct rte_ipv4_hdr *)template;
1336                 /*
1337                  * Offset of IPv4 length field and its initial value
1338                  * (IP + UDP + VXLAN) are specified in the action. The NIC
1339                  * will add inner packet length.
1340                  */
1341                 fm_op.encap.len1_offset = off +
1342                         offsetof(struct rte_ipv4_hdr, total_length);
1343                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1344                         sizeof(struct rte_udp_hdr) +
1345                         sizeof(struct rte_vxlan_hdr);
1346                 append_template(&template, &off, item->spec,
1347                                 sizeof(struct rte_ipv4_hdr));
1348                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1349                 if (ip4->time_to_live == 0)
1350                         ip4->time_to_live = IP_DEFTTL;
1351                 ip4->next_proto_id = IPPROTO_UDP;
1352                 break;
1353         }
1354         case RTE_FLOW_ITEM_TYPE_IPV6:
1355         {
1356                 struct rte_ipv6_hdr *ip6;
1357
1358                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1359                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1360                 ip6 = (struct rte_ipv6_hdr *)template;
1361                 fm_op.encap.len1_offset = off +
1362                         offsetof(struct rte_ipv6_hdr, payload_len);
1363                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1364                         sizeof(struct rte_vxlan_hdr);
1365                 append_template(&template, &off, item->spec,
1366                                 sizeof(struct rte_ipv6_hdr));
1367                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1368                 if (ip6->hop_limits == 0)
1369                         ip6->hop_limits = IP_DEFTTL;
1370                 ip6->proto = IPPROTO_UDP;
1371                 break;
1372         }
1373         default:
1374                 return rte_flow_error_set(error,
1375                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1376                         "vxlan-encap: L3 must be IPv4/IPv6");
1377         }
1378         item++;
1379         flow_item_skip_void(&item);
1380
1381         /* L4 is UDP */
1382         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1383                 return rte_flow_error_set(error, EINVAL,
1384                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1385                         "vxlan-encap: UDP must follow IPv4/IPv6");
1386         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1387         fm_op.encap.len2_offset =
1388                 off + offsetof(struct rte_udp_hdr, dgram_len);
1389         fm_op.encap.len2_delta =
1390                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1391         udp = (struct rte_udp_hdr *)template;
1392         append_template(&template, &off, item->spec,
1393                         sizeof(struct rte_udp_hdr));
1394         /*
1395          * Firmware does not hash/fill source port yet. Generate a
1396          * random port, as there is *usually* one rte_flow for the
1397          * given inner packet stream (i.e. a single stream has one
1398          * random port).
1399          */
1400         if (udp->src_port == 0)
1401                 udp->src_port = gen_src_port();
1402         item++;
1403         flow_item_skip_void(&item);
1404
1405         /* Finally VXLAN */
1406         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1407                 return rte_flow_error_set(error,
1408                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1409                         "vxlan-encap: VXLAN must follow UDP");
1410         append_template(&template, &off, item->spec,
1411                         sizeof(struct rte_flow_item_vxlan));
1412
1413         /*
1414          * Fill in the rest of the action structure.
1415          * Indicate that we want to encap with vxlan at packet start.
1416          */
1417         fm_op.encap.template_offset = 0;
1418         fm_op.encap.template_len = off;
1419         return enic_fm_append_action_op(fm, &fm_op, error);
1420 }
1421
1422 static int
1423 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1424                   uint64_t *handle)
1425 {
1426         uint32_t bdf;
1427         uint64_t args[2];
1428         int rc;
1429
1430         ENICPMD_FUNC_TRACE();
1431         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1432                     addr->function);
1433         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1434         args[0] = FM_VNIC_FIND;
1435         args[1] = bdf;
1436         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1437         if (rc != 0) {
1438                 /* Expected to fail if BDF is not on the adapter */
1439                 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1440                 return rc;
1441         }
1442         *handle = args[0];
1443         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1444         return 0;
1445 }
1446
1447 /*
1448  * Egress: target port should be either PF uplink or VF.
1449  * Supported cases
1450  * 1. VF egress -> PF uplink
1451  *   PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1452  * 2. VF egress -> VF
1453  *
1454  * Unsupported cases
1455  * 1. PF egress -> VF
1456  *   App should be using representor to pass packets to VF
1457  */
1458 static int
1459 vf_egress_port_id_action(struct enic_flowman *fm,
1460                          struct rte_eth_dev *dst_dev,
1461                          uint64_t dst_vnic_h,
1462                          struct fm_action_op *fm_op,
1463                          struct rte_flow_error *error)
1464 {
1465         struct enic *src_enic, *dst_enic;
1466         struct enic_vf_representor *vf;
1467         uint8_t uif;
1468         int ret;
1469
1470         ENICPMD_FUNC_TRACE();
1471         src_enic = fm->user_enic;
1472         dst_enic = pmd_priv(dst_dev);
1473         if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1474                 return rte_flow_error_set(error, EINVAL,
1475                         RTE_FLOW_ERROR_TYPE_ACTION,
1476                         NULL, "source port is not VF representor");
1477         }
1478
1479         /* VF -> PF uplink. dst is not VF representor */
1480         if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1481                 /* PF is the VF's PF? Then nothing to do */
1482                 vf = VF_ENIC_TO_VF_REP(src_enic);
1483                 if (vf->pf == dst_enic) {
1484                         ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1485                         return 0;
1486                 }
1487                 /* If not, steer to the remote PF's uplink */
1488                 uif = dst_enic->fm_vnic_uif;
1489                 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1490                 memset(fm_op, 0, sizeof(*fm_op));
1491                 fm_op->fa_op = FMOP_SET_EGPORT;
1492                 fm_op->set_egport.egport = uif;
1493                 ret = enic_fm_append_action_op(fm, fm_op, error);
1494                 return ret;
1495         }
1496
1497         /* VF -> VF loopback. Hairpin and steer to vnic */
1498         memset(fm_op, 0, sizeof(*fm_op));
1499         fm_op->fa_op = FMOP_EG_HAIRPIN;
1500         ret = enic_fm_append_action_op(fm, fm_op, error);
1501         if (ret)
1502                 return ret;
1503         ENICPMD_LOG(DEBUG, "egress hairpin");
1504         fm->hairpin_steer_vnic_h = dst_vnic_h;
1505         fm->need_hairpin_steer = 1;
1506         return 0;
1507 }
1508
1509 static int
1510 enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
1511                            struct rte_eth_dev **dst_dev,
1512                            struct rte_flow_error *error)
1513 {
1514         struct rte_eth_dev *dev;
1515
1516         ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
1517         if (!rte_eth_dev_is_valid_port(dst_port_id)) {
1518                 return rte_flow_error_set(error, EINVAL,
1519                         RTE_FLOW_ERROR_TYPE_ACTION,
1520                         NULL, "invalid port_id");
1521         }
1522         dev = &rte_eth_devices[dst_port_id];
1523         if (!dev_is_enic(dev)) {
1524                 return rte_flow_error_set(error, EINVAL,
1525                         RTE_FLOW_ERROR_TYPE_ACTION,
1526                         NULL, "port_id is not enic");
1527         }
1528         if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
1529                 return rte_flow_error_set(error, EINVAL,
1530                         RTE_FLOW_ERROR_TYPE_ACTION,
1531                         NULL, "destination and source ports are not in the same switch domain");
1532         }
1533
1534         *dst_dev = dev;
1535         return 0;
1536 }
1537
1538 /* Translate flow actions to flowman TCAM entry actions */
1539 static int
1540 enic_fm_copy_action(struct enic_flowman *fm,
1541                     const struct rte_flow_action actions[],
1542                     uint8_t ingress,
1543                     struct rte_flow_error *error)
1544 {
1545         enum {
1546                 FATE = 1 << 0,
1547                 DECAP = 1 << 1,
1548                 PASSTHRU = 1 << 2,
1549                 COUNT = 1 << 3,
1550                 ENCAP = 1 << 4,
1551                 PUSH_VLAN = 1 << 5,
1552                 PORT_ID = 1 << 6,
1553         };
1554         struct fm_tcam_match_entry *fmt;
1555         struct fm_action_op fm_op;
1556         bool need_ovlan_action;
1557         struct enic *enic;
1558         uint32_t overlap;
1559         uint64_t vnic_h;
1560         uint16_t ovlan;
1561         bool first_rq;
1562         bool steer;
1563         int ret;
1564
1565         ENICPMD_FUNC_TRACE();
1566         fmt = &fm->tcam_entry;
1567         need_ovlan_action = false;
1568         ovlan = 0;
1569         first_rq = true;
1570         steer = false;
1571         enic = fm->user_enic;
1572         overlap = 0;
1573         vnic_h = enic->fm_vnic_handle;
1574
1575         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1576                 switch (actions->type) {
1577                 case RTE_FLOW_ACTION_TYPE_VOID:
1578                         continue;
1579                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1580                         if (overlap & PASSTHRU)
1581                                 goto unsupported;
1582                         overlap |= PASSTHRU;
1583                         break;
1584                 }
1585                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1586                         const struct rte_flow_action_jump *jump =
1587                                 actions->conf;
1588                         struct enic_fm_fet *fet;
1589
1590                         if (overlap & FATE)
1591                                 goto unsupported;
1592                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1593                                            &fet, error);
1594                         if (ret)
1595                                 return ret;
1596                         overlap |= FATE;
1597                         memset(&fm_op, 0, sizeof(fm_op));
1598                         fm_op.fa_op = FMOP_EXACT_MATCH;
1599                         fm_op.exact.handle = fet->handle;
1600                         fm->fet = fet;
1601                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1602                         if (ret)
1603                                 return ret;
1604                         break;
1605                 }
1606                 case RTE_FLOW_ACTION_TYPE_MARK: {
1607                         const struct rte_flow_action_mark *mark =
1608                                 actions->conf;
1609
1610                         if (enic->use_noscatter_vec_rx_handler)
1611                                 goto unsupported;
1612                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1613                                 return rte_flow_error_set(error, EINVAL,
1614                                         RTE_FLOW_ERROR_TYPE_ACTION,
1615                                         NULL, "invalid mark id");
1616                         memset(&fm_op, 0, sizeof(fm_op));
1617                         fm_op.fa_op = FMOP_MARK;
1618                         fm_op.mark.mark = mark->id + 1;
1619                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1620                         if (ret)
1621                                 return ret;
1622                         break;
1623                 }
1624                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1625                         if (enic->use_noscatter_vec_rx_handler)
1626                                 goto unsupported;
1627                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1628                         memset(&fm_op, 0, sizeof(fm_op));
1629                         fm_op.fa_op = FMOP_MARK;
1630                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1631                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1632                         if (ret)
1633                                 return ret;
1634                         break;
1635                 }
1636                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1637                         const struct rte_flow_action_queue *queue =
1638                                 actions->conf;
1639
1640                         /*
1641                          * If fate other than QUEUE or RSS, fail. Multiple
1642                          * rss and queue actions are ok.
1643                          */
1644                         if ((overlap & FATE) && first_rq)
1645                                 goto unsupported;
1646                         first_rq = false;
1647                         overlap |= FATE;
1648                         memset(&fm_op, 0, sizeof(fm_op));
1649                         fm_op.fa_op = FMOP_RQ_STEER;
1650                         fm_op.rq_steer.rq_index =
1651                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1652                         fm_op.rq_steer.rq_count = 1;
1653                         fm_op.rq_steer.vnic_handle = vnic_h;
1654                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1655                         if (ret)
1656                                 return ret;
1657                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1658                                     fm_op.rq_steer.rq_index);
1659                         steer = true;
1660                         break;
1661                 }
1662                 case RTE_FLOW_ACTION_TYPE_DROP: {
1663                         if (overlap & FATE)
1664                                 goto unsupported;
1665                         overlap |= FATE;
1666                         memset(&fm_op, 0, sizeof(fm_op));
1667                         fm_op.fa_op = FMOP_DROP;
1668                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1669                         if (ret)
1670                                 return ret;
1671                         ENICPMD_LOG(DEBUG, "create DROP action");
1672                         break;
1673                 }
1674                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1675                         if (overlap & COUNT)
1676                                 goto unsupported;
1677                         overlap |= COUNT;
1678                         /* Count is associated with entry not action on VIC. */
1679                         fmt->ftm_flags |= FMEF_COUNTER;
1680                         break;
1681                 }
1682                 case RTE_FLOW_ACTION_TYPE_RSS: {
1683                         const struct rte_flow_action_rss *rss = actions->conf;
1684                         bool allow;
1685                         uint16_t i;
1686
1687                         /*
1688                          * If fate other than QUEUE or RSS, fail. Multiple
1689                          * rss and queue actions are ok.
1690                          */
1691                         if ((overlap & FATE) && first_rq)
1692                                 goto unsupported;
1693                         first_rq = false;
1694                         overlap |= FATE;
1695
1696                         /*
1697                          * Hardware only supports RSS actions on outer level
1698                          * with default type and function. Queues must be
1699                          * sequential.
1700                          */
1701                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1702                                 rss->level == 0 && (rss->types == 0 ||
1703                                 rss->types == enic->rss_hf) &&
1704                                 rss->queue_num <= enic->rq_count &&
1705                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1706
1707
1708                         /* Identity queue map needs to be sequential */
1709                         for (i = 1; i < rss->queue_num; i++)
1710                                 allow = allow && (rss->queue[i] ==
1711                                         rss->queue[i - 1] + 1);
1712                         if (!allow)
1713                                 goto unsupported;
1714
1715                         memset(&fm_op, 0, sizeof(fm_op));
1716                         fm_op.fa_op = FMOP_RQ_STEER;
1717                         fm_op.rq_steer.rq_index =
1718                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1719                         fm_op.rq_steer.rq_count = rss->queue_num;
1720                         fm_op.rq_steer.vnic_handle = vnic_h;
1721                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1722                         if (ret)
1723                                 return ret;
1724                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1725                                     fm_op.rq_steer.rq_index);
1726                         steer = true;
1727                         break;
1728                 }
1729                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1730                         const struct rte_flow_action_port_id *port;
1731                         struct rte_eth_dev *dev = NULL;
1732
1733                         if (!ingress && (overlap & PORT_ID)) {
1734                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1735                                 goto unsupported;
1736                         }
1737                         port = actions->conf;
1738                         if (port->original) {
1739                                 vnic_h = enic->fm_vnic_handle; /* This port */
1740                                 break;
1741                         }
1742                         ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
1743                                                          error);
1744                         if (ret)
1745                                 return ret;
1746                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1747                         overlap |= PORT_ID;
1748                         /*
1749                          * Ingress. Nothing more to do. We add an implicit
1750                          * steer at the end if needed.
1751                          */
1752                         if (ingress)
1753                                 break;
1754                         /* Egress */
1755                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1756                                 error);
1757                         if (ret)
1758                                 return ret;
1759                         break;
1760                 }
1761                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1762                         if (overlap & DECAP)
1763                                 goto unsupported;
1764                         overlap |= DECAP;
1765
1766                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1767                                 error);
1768                         if (ret != 0)
1769                                 return ret;
1770                         break;
1771                 }
1772                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1773                         const struct rte_flow_action_vxlan_encap *encap;
1774
1775                         encap = actions->conf;
1776                         if (overlap & ENCAP)
1777                                 goto unsupported;
1778                         overlap |= ENCAP;
1779                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1780                                 error);
1781                         if (ret != 0)
1782                                 return ret;
1783                         break;
1784                 }
1785                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1786                         struct fm_action_op *decap;
1787
1788                         /*
1789                          * If decap-nostrip appears before pop vlan, this pop
1790                          * applies to the inner packet vlan. Turn it into
1791                          * decap-strip.
1792                          */
1793                         decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1794                         if (decap) {
1795                                 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1796                                 decap->fa_op = FMOP_DECAP_STRIP;
1797                                 break;
1798                         }
1799                         memset(&fm_op, 0, sizeof(fm_op));
1800                         fm_op.fa_op = FMOP_POP_VLAN;
1801                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1802                         if (ret)
1803                                 return ret;
1804                         break;
1805                 }
1806                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1807                         const struct rte_flow_action_of_push_vlan *vlan;
1808
1809                         if (overlap & PASSTHRU)
1810                                 goto unsupported;
1811                         vlan = actions->conf;
1812                         if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1813                                 return rte_flow_error_set(error, EINVAL,
1814                                         RTE_FLOW_ERROR_TYPE_ACTION,
1815                                         NULL, "unexpected push_vlan ethertype");
1816                         }
1817                         overlap |= PUSH_VLAN;
1818                         need_ovlan_action = true;
1819                         break;
1820                 }
1821                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1822                         const struct rte_flow_action_of_set_vlan_pcp *pcp;
1823
1824                         pcp = actions->conf;
1825                         if (pcp->vlan_pcp > 7) {
1826                                 return rte_flow_error_set(error, EINVAL,
1827                                         RTE_FLOW_ERROR_TYPE_ACTION,
1828                                         NULL, "invalid vlan_pcp");
1829                         }
1830                         need_ovlan_action = true;
1831                         ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1832                         break;
1833                 }
1834                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1835                         const struct rte_flow_action_of_set_vlan_vid *vid;
1836
1837                         vid = actions->conf;
1838                         need_ovlan_action = true;
1839                         ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1840                         break;
1841                 }
1842                 case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
1843                         const struct rte_flow_action_ethdev *ethdev;
1844                         struct rte_eth_dev *dev = NULL;
1845
1846                         ethdev = actions->conf;
1847                         ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1848                                                          &dev, error);
1849                         if (ret)
1850                                 return ret;
1851                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1852                         overlap |= PORT_ID;
1853                         /*
1854                          * Action PORT_REPRESENTOR implies ingress destination.
1855                          * Noting to do. We add an implicit stree at the
1856                          * end if needed.
1857                          */
1858                         ingress = 1;
1859                         break;
1860                 }
1861                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
1862                         const struct rte_flow_action_ethdev *ethdev;
1863                         struct rte_eth_dev *dev = NULL;
1864
1865                         if (overlap & PORT_ID) {
1866                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1867                                 goto unsupported;
1868                         }
1869                         ethdev = actions->conf;
1870                         ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1871                                                          &dev, error);
1872                         if (ret)
1873                                 return ret;
1874                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1875                         overlap |= PORT_ID;
1876                         /* Action REPRESENTED_PORT: always egress destination */
1877                         ingress = 0;
1878                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1879                                 error);
1880                         if (ret)
1881                                 return ret;
1882                         break;
1883                 }
1884                 default:
1885                         goto unsupported;
1886                 }
1887         }
1888
1889         if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1890                 goto unsupported;
1891         /* Egress from VF: need implicit WQ match */
1892         if (enic_is_vf_rep(enic) && !ingress) {
1893                 fmt->ftm_data.fk_wq_id = 0;
1894                 fmt->ftm_mask.fk_wq_id = 0xffff;
1895                 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1896                 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1897                             VF_ENIC_TO_VF_REP(enic)->vf_id);
1898         }
1899         if (need_ovlan_action) {
1900                 memset(&fm_op, 0, sizeof(fm_op));
1901                 fm_op.fa_op = FMOP_SET_OVLAN;
1902                 fm_op.ovlan.vlan = ovlan;
1903                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1904                 if (ret)
1905                         return ret;
1906         }
1907         /* Add steer op for PORT_ID without QUEUE */
1908         if ((overlap & PORT_ID) && !steer && ingress) {
1909                 memset(&fm_op, 0, sizeof(fm_op));
1910                 /* Always to queue 0 for now as generic RSS is not available */
1911                 fm_op.fa_op = FMOP_RQ_STEER;
1912                 fm_op.rq_steer.rq_index = 0;
1913                 fm_op.rq_steer.vnic_handle = vnic_h;
1914                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1915                 if (ret)
1916                         return ret;
1917                 ENICPMD_LOG(DEBUG, "add implicit steer op");
1918         }
1919         /* Add required END */
1920         memset(&fm_op, 0, sizeof(fm_op));
1921         fm_op.fa_op = FMOP_END;
1922         ret = enic_fm_append_action_op(fm, &fm_op, error);
1923         if (ret)
1924                 return ret;
1925         enic_fm_reorder_action_op(fm);
1926         return 0;
1927
1928 unsupported:
1929         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1930                                   NULL, "enic: unsupported action");
1931 }
1932
1933 /** Check if the action is supported */
1934 static int
1935 enic_fm_match_action(const struct rte_flow_action *action,
1936                      const enum rte_flow_action_type *supported_actions)
1937 {
1938         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1939              supported_actions++) {
1940                 if (action->type == *supported_actions)
1941                         return 1;
1942         }
1943         return 0;
1944 }
1945
1946 /* Debug function to dump internal NIC action structure. */
1947 static void
1948 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1949 {
1950         /* Manually keep in sync with FMOP commands */
1951         const char *fmop_str[FMOP_OP_MAX] = {
1952                 [FMOP_END] = "end",
1953                 [FMOP_DROP] = "drop",
1954                 [FMOP_RQ_STEER] = "steer",
1955                 [FMOP_EXACT_MATCH] = "exmatch",
1956                 [FMOP_MARK] = "mark",
1957                 [FMOP_EXT_MARK] = "ext_mark",
1958                 [FMOP_TAG] = "tag",
1959                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1960                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1961                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1962                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1963                 [FMOP_ENCAP] = "encap",
1964                 [FMOP_SET_OVLAN] = "set_ovlan",
1965                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1966                 [FMOP_DECAP_STRIP] = "decap_strip",
1967                 [FMOP_POP_VLAN] = "pop_vlan",
1968                 [FMOP_SET_EGPORT] = "set_egport",
1969                 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1970                 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1971                 [FMOP_EMIT] = "emit",
1972                 [FMOP_MODIFY] = "modify",
1973         };
1974         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1975         char buf[128], *bp = buf;
1976         const char *op_str;
1977         int i, n, buf_len;
1978
1979         buf[0] = '\0';
1980         buf_len = sizeof(buf);
1981         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1982                 if (op->fa_op == FMOP_END)
1983                         break;
1984                 if (op->fa_op >= FMOP_OP_MAX)
1985                         op_str = "unknown";
1986                 else
1987                         op_str = fmop_str[op->fa_op];
1988                 n = snprintf(bp, buf_len, "%s,", op_str);
1989                 if (n > 0 && n < buf_len) {
1990                         bp += n;
1991                         buf_len -= n;
1992                 }
1993                 op++;
1994         }
1995         /* Remove trailing comma */
1996         if (buf[0])
1997                 *(bp - 1) = '\0';
1998         ENICPMD_LOG(DEBUG, "       Actions: %s", buf);
1999 }
2000
2001 static int
2002 bits_to_str(uint32_t bits, const char *strings[], int max,
2003             char *buf, int buf_len)
2004 {
2005         int i, n = 0, len = 0;
2006
2007         for (i = 0; i < max; i++) {
2008                 if (bits & (1 << i)) {
2009                         n = snprintf(buf, buf_len, "%s,", strings[i]);
2010                         if (n > 0 && n < buf_len) {
2011                                 buf += n;
2012                                 buf_len -= n;
2013                                 len += n;
2014                         }
2015                 }
2016         }
2017         /* Remove trailing comma */
2018         if (len) {
2019                 *(buf - 1) = '\0';
2020                 len--;
2021         }
2022         return len;
2023 }
2024
2025 /* Debug function to dump internal NIC filter structure. */
2026 static void
2027 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
2028                           int buf_len)
2029 {
2030         /* Manually keep in sync with FKM_BITS */
2031         const char *fm_fkm_str[FKM_BIT_COUNT] = {
2032                 [FKM_QTAG_BIT] = "qtag",
2033                 [FKM_CMD_BIT] = "cmd",
2034                 [FKM_IPV4_BIT] = "ip4",
2035                 [FKM_IPV6_BIT] = "ip6",
2036                 [FKM_ROCE_BIT] = "roce",
2037                 [FKM_UDP_BIT] = "udp",
2038                 [FKM_TCP_BIT] = "tcp",
2039                 [FKM_TCPORUDP_BIT] = "tcpportudp",
2040                 [FKM_IPFRAG_BIT] = "ipfrag",
2041                 [FKM_NVGRE_BIT] = "nvgre",
2042                 [FKM_VXLAN_BIT] = "vxlan",
2043                 [FKM_GENEVE_BIT] = "geneve",
2044                 [FKM_NSH_BIT] = "nsh",
2045                 [FKM_ROCEV2_BIT] = "rocev2",
2046                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
2047                 [FKM_IPOK_BIT] = "ipok",
2048                 [FKM_L4OK_BIT] = "l4ok",
2049                 [FKM_ROCEOK_BIT] = "roceok",
2050                 [FKM_FCSOK_BIT] = "fcsok",
2051                 [FKM_EG_SPAN_BIT] = "eg_span",
2052                 [FKM_IG_SPAN_BIT] = "ig_span",
2053                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
2054         };
2055         /* Manually keep in sync with FKH_BITS */
2056         const char *fm_fkh_str[FKH_BIT_COUNT] = {
2057                 [FKH_ETHER_BIT] = "eth",
2058                 [FKH_QTAG_BIT] = "qtag",
2059                 [FKH_L2RAW_BIT] = "l2raw",
2060                 [FKH_IPV4_BIT] = "ip4",
2061                 [FKH_IPV6_BIT] = "ip6",
2062                 [FKH_L3RAW_BIT] = "l3raw",
2063                 [FKH_UDP_BIT] = "udp",
2064                 [FKH_TCP_BIT] = "tcp",
2065                 [FKH_ICMP_BIT] = "icmp",
2066                 [FKH_VXLAN_BIT] = "vxlan",
2067                 [FKH_L4RAW_BIT] = "l4raw",
2068         };
2069         uint32_t fkh_bits = fk_hdrset->fk_header_select;
2070         uint32_t fkm_bits = fk_hdrset->fk_metadata;
2071         int n;
2072
2073         if (!fkm_bits && !fkh_bits)
2074                 return;
2075         n = snprintf(buf, buf_len, "metadata(");
2076         if (n > 0 && n < buf_len) {
2077                 buf += n;
2078                 buf_len -= n;
2079         }
2080         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
2081         if (n > 0 && n < buf_len) {
2082                 buf += n;
2083                 buf_len -= n;
2084         }
2085         n = snprintf(buf, buf_len, ") valid hdr fields(");
2086         if (n > 0 && n < buf_len) {
2087                 buf += n;
2088                 buf_len -= n;
2089         }
2090         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
2091         if (n > 0 && n < buf_len) {
2092                 buf += n;
2093                 buf_len -= n;
2094         }
2095         snprintf(buf, buf_len, ")");
2096 }
2097
2098 static void
2099 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
2100                         uint8_t ingress)
2101 {
2102         char buf[256];
2103
2104         memset(buf, 0, sizeof(buf));
2105         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
2106                                   buf, sizeof(buf));
2107         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
2108                     (ingress) ? "IG" : "EG", buf,
2109                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
2110                     match->ftm_position);
2111         memset(buf, 0, sizeof(buf));
2112         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
2113                                   buf, sizeof(buf));
2114         if (buf[0])
2115                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
2116 }
2117
2118 /* Debug function to dump internal NIC flow structures. */
2119 static void
2120 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
2121                         const struct fm_action *fm_action,
2122                         uint8_t ingress)
2123 {
2124         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
2125                 return;
2126         enic_fm_dump_tcam_match(fm_match, ingress);
2127         enic_fm_dump_tcam_actions(fm_action);
2128 }
2129
2130 static int
2131 enic_fm_flow_parse(struct enic_flowman *fm,
2132                    const struct rte_flow_attr *attrs,
2133                    const struct rte_flow_item pattern[],
2134                    const struct rte_flow_action actions[],
2135                    struct rte_flow_error *error)
2136 {
2137         const struct rte_flow_action *action;
2138         unsigned int ret;
2139         static const enum rte_flow_action_type *sa;
2140
2141         ENICPMD_FUNC_TRACE();
2142         ret = 0;
2143         if (!pattern) {
2144                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
2145                                    NULL, "no pattern specified");
2146                 return -rte_errno;
2147         }
2148
2149         if (!actions) {
2150                 rte_flow_error_set(error, EINVAL,
2151                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
2152                                    NULL, "no action specified");
2153                 return -rte_errno;
2154         }
2155
2156         if (attrs) {
2157                 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
2158                         rte_flow_error_set(error, ENOTSUP,
2159                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2160                                            NULL,
2161                                            "priorities are not supported for non-default (0) groups");
2162                         return -rte_errno;
2163                 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
2164                         rte_flow_error_set(error, ENOTSUP,
2165                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2166                                            NULL,
2167                                            "transfer is not supported");
2168                         return -rte_errno;
2169                 } else if (attrs->ingress && attrs->egress) {
2170                         rte_flow_error_set(error, ENOTSUP,
2171                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2172                                            NULL,
2173                                            "bidirectional rules not supported");
2174                         return -rte_errno;
2175                 }
2176
2177         } else {
2178                 rte_flow_error_set(error, EINVAL,
2179                                    RTE_FLOW_ERROR_TYPE_ATTR,
2180                                    NULL, "no attribute specified");
2181                 return -rte_errno;
2182         }
2183
2184         /* Verify Actions. */
2185         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
2186              enic_fm_supported_eg_actions;
2187         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
2188              action++) {
2189                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
2190                         continue;
2191                 else if (!enic_fm_match_action(action, sa))
2192                         break;
2193         }
2194         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
2195                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
2196                                    action, "invalid action");
2197                 return -rte_errno;
2198         }
2199         ret = enic_fm_copy_entry(fm, pattern, error);
2200         if (ret)
2201                 return ret;
2202         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
2203         return ret;
2204 }
2205
2206 static void
2207 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2208 {
2209         if (!fm_flow->counter_valid)
2210                 return;
2211         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
2212         fm_flow->counter_valid = false;
2213 }
2214
2215 static int
2216 enic_fm_more_counters(struct enic_flowman *fm)
2217 {
2218         struct enic_fm_counter *new_stack;
2219         struct enic_fm_counter *ctrs;
2220         int i, rc;
2221         uint64_t args[2];
2222
2223         ENICPMD_FUNC_TRACE();
2224         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
2225                                 FM_COUNTERS_EXPAND) *
2226                                 sizeof(struct enic_fm_counter), 0);
2227         if (new_stack == NULL) {
2228                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
2229                 return -ENOMEM;
2230         }
2231         fm->counter_stack = new_stack;
2232
2233         args[0] = FM_COUNTER_BRK;
2234         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
2235         rc = flowman_cmd(fm, args, 2);
2236         if (rc != 0) {
2237                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
2238                 return rc;
2239         }
2240         ctrs = (struct enic_fm_counter *)fm->counter_stack +
2241                 fm->counters_alloced;
2242         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
2243                 ctrs->handle = fm->counters_alloced + i;
2244                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
2245         }
2246         fm->counters_alloced += FM_COUNTERS_EXPAND;
2247         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
2248                     FM_COUNTERS_EXPAND, fm->counters_alloced);
2249         return 0;
2250 }
2251
2252 static int
2253 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
2254 {
2255         uint64_t args[3];
2256         int ret;
2257
2258         ENICPMD_FUNC_TRACE();
2259         args[0] = FM_COUNTER_QUERY;
2260         args[1] = c->handle;
2261         args[2] = 1; /* clear */
2262         ret = flowman_cmd(fm, args, 3);
2263         if (ret) {
2264                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
2265                             ret, c->handle);
2266                 return ret;
2267         }
2268         return 0;
2269 }
2270
2271 static int
2272 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
2273                       struct enic_fm_counter **ctr)
2274 {
2275         struct enic_fm_counter *c;
2276         int ret;
2277
2278         ENICPMD_FUNC_TRACE();
2279         *ctr = NULL;
2280         if (SLIST_EMPTY(&fm->counters)) {
2281                 ret = enic_fm_more_counters(fm);
2282                 if (ret)
2283                         return rte_flow_error_set(error, -ret,
2284                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2285                                 NULL, "enic: out of counters");
2286         }
2287         c = SLIST_FIRST(&fm->counters);
2288         SLIST_REMOVE_HEAD(&fm->counters, next);
2289         *ctr = c;
2290         return 0;
2291 }
2292
2293 static int
2294 enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
2295 {
2296         uint64_t args[2];
2297         int ret = 0;
2298
2299         ENICPMD_FUNC_TRACE();
2300         RTE_ASSERT(ah->ref > 0);
2301         ah->ref--;
2302         if (ah->ref == 0) {
2303                 args[0] = FM_ACTION_FREE;
2304                 args[1] = ah->handle;
2305                 ret = flowman_cmd(fm, args, 2);
2306                 if (ret)
2307                         /* This is a "should never happen" error. */
2308                         ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
2309                                     PRIx64, ret, ah->handle);
2310                 rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
2311                 free(ah);
2312         }
2313         return ret;
2314 }
2315
2316 static int
2317 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
2318 {
2319         uint64_t args[2];
2320         int rc;
2321
2322         ENICPMD_FUNC_TRACE();
2323         args[0] = FM_MATCH_ENTRY_REMOVE;
2324         args[1] = handle;
2325         rc = flowman_cmd(fm, args, 2);
2326         if (rc)
2327                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
2328                             " handle=0x%" PRIx64, rc, handle);
2329         return rc;
2330 }
2331
2332 static struct enic_fm_jump_flow *
2333 find_jump_flow(struct enic_flowman *fm, uint32_t group)
2334 {
2335         struct enic_fm_jump_flow *j;
2336
2337         ENICPMD_FUNC_TRACE();
2338         TAILQ_FOREACH(j, &fm->jump_list, list) {
2339                 if (j->group == group)
2340                         return j;
2341         }
2342         return NULL;
2343 }
2344
2345 static void
2346 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2347 {
2348         struct enic_fm_jump_flow *j;
2349
2350         ENICPMD_FUNC_TRACE();
2351         TAILQ_FOREACH(j, &fm->jump_list, list) {
2352                 if (j->flow == flow) {
2353                         TAILQ_REMOVE(&fm->jump_list, j, list);
2354                         free(j);
2355                         return;
2356                 }
2357         }
2358 }
2359
2360 static int
2361 save_jump_flow(struct enic_flowman *fm,
2362                struct rte_flow *flow,
2363                uint32_t group,
2364                struct fm_tcam_match_entry *match,
2365                struct fm_action *action)
2366 {
2367         struct enic_fm_jump_flow *j;
2368
2369         ENICPMD_FUNC_TRACE();
2370         j = calloc(1, sizeof(struct enic_fm_jump_flow));
2371         if (j == NULL)
2372                 return -ENOMEM;
2373         j->flow = flow;
2374         j->group = group;
2375         j->match = *match;
2376         j->action = *action;
2377         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2378         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2379         return 0;
2380 }
2381
2382 static void
2383 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2384 {
2385         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2386                 enic_fm_entry_free(fm, fm_flow->entry_handle);
2387                 fm_flow->entry_handle = FM_INVALID_HANDLE;
2388         }
2389         if (fm_flow->action != NULL) {
2390                 enic_fm_action_free(fm, fm_flow->action);
2391                 fm_flow->action = NULL;
2392         }
2393         enic_fm_counter_free(fm, fm_flow);
2394         if (fm_flow->fet) {
2395                 enic_fet_put(fm, fm_flow->fet);
2396                 fm_flow->fet = NULL;
2397         }
2398 }
2399
2400 static void
2401 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2402 {
2403         struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2404
2405         if (flow->fm->fet && flow->fm->fet->default_key)
2406                 remove_jump_flow(fm, flow);
2407         __enic_fm_flow_free(fm, flow->fm);
2408         if (steer) {
2409                 __enic_fm_flow_free(fm, steer);
2410                 free(steer);
2411         }
2412         free(flow->fm);
2413         free(flow);
2414 }
2415
2416 static int
2417 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2418                        struct fm_tcam_match_entry *match_in,
2419                        uint64_t *entry_handle,
2420                        uint8_t ingress,
2421                        struct rte_flow_error *error)
2422 {
2423         struct fm_tcam_match_entry *ftm;
2424         uint64_t args[3];
2425         int ret;
2426
2427         ENICPMD_FUNC_TRACE();
2428         /* Copy entry to the command buffer */
2429         ftm = &fm->cmd.va->fm_tcam_match_entry;
2430         memcpy(ftm, match_in, sizeof(*ftm));
2431         /* Add TCAM entry */
2432         args[0] = FM_TCAM_ENTRY_INSTALL;
2433         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2434         args[2] = fm->cmd.pa;
2435         ret = flowman_cmd(fm, args, 3);
2436         if (ret != 0) {
2437                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2438                             ingress ? "ingress" : "egress", ret);
2439                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2440                         NULL, "enic: devcmd(tcam-entry-install)");
2441                 return ret;
2442         }
2443         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2444                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
2445         *entry_handle = args[0];
2446         return 0;
2447 }
2448
2449 static int
2450 enic_fm_add_exact_entry(struct enic_flowman *fm,
2451                         struct fm_tcam_match_entry *match_in,
2452                         uint64_t *entry_handle,
2453                         struct enic_fm_fet *fet,
2454                         struct rte_flow_error *error)
2455 {
2456         struct fm_exact_match_entry *fem;
2457         uint64_t args[3];
2458         int ret;
2459
2460         ENICPMD_FUNC_TRACE();
2461         /* The new entry must have the table's key */
2462         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2463                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2464                 return rte_flow_error_set(error, EINVAL,
2465                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2466                         "enic: key does not match group's key");
2467         }
2468
2469         /* Copy entry to the command buffer */
2470         fem = &fm->cmd.va->fm_exact_match_entry;
2471         /*
2472          * Translate TCAM entry to exact entry. As is only need to drop
2473          * position and mask. The mask is part of the exact match table.
2474          * Position (aka priority) is not supported in the exact match table.
2475          */
2476         fem->fem_data = match_in->ftm_data;
2477         fem->fem_flags = match_in->ftm_flags;
2478         fem->fem_action = match_in->ftm_action;
2479         fem->fem_counter = match_in->ftm_counter;
2480
2481         /* Add exact entry */
2482         args[0] = FM_EXACT_ENTRY_INSTALL;
2483         args[1] = fet->handle;
2484         args[2] = fm->cmd.pa;
2485         ret = flowman_cmd(fm, args, 3);
2486         if (ret != 0) {
2487                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2488                             fet->ingress ? "ingress" : "egress", fet->group);
2489                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2490                         NULL, "enic: devcmd(exact-entry-install)");
2491                 return ret;
2492         }
2493         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2494                     " handle=0x%" PRIx64,
2495                     fet->ingress ? "ingress" : "egress", fet->group,
2496                     (uint64_t)args[0]);
2497         *entry_handle = args[0];
2498         return 0;
2499 }
2500
2501 static int
2502 enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
2503                        struct rte_flow_error *error,
2504                        struct enic_fm_action **ah_o)
2505 {
2506         struct enic_fm_action *ah;
2507         struct fm_action *fma;
2508         uint64_t args[2];
2509         int ret = 0;
2510
2511         ret = rte_hash_lookup_data(fm->action_hash, action_in,
2512                                    (void **)&ah);
2513         if (ret < 0 && ret != -ENOENT)
2514                 return rte_flow_error_set(error, -ret,
2515                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2516                                    NULL, "enic: rte_hash_lookup(action)");
2517
2518         if (ret == -ENOENT) {
2519                 /* Allocate a new action on the NIC. */
2520                 fma = &fm->cmd.va->fm_action;
2521                 memcpy(fma, action_in, sizeof(*fma));
2522
2523                 ah = calloc(1, sizeof(*ah));
2524                 memcpy(&ah->key, action_in, sizeof(struct fm_action));
2525                 if (ah == NULL)
2526                         return rte_flow_error_set(error, ENOMEM,
2527                                            RTE_FLOW_ERROR_TYPE_HANDLE,
2528                                            NULL, "enic: calloc(fm-action)");
2529                 args[0] = FM_ACTION_ALLOC;
2530                 args[1] = fm->cmd.pa;
2531                 ret = flowman_cmd(fm, args, 2);
2532                 if (ret != 0) {
2533                         rte_flow_error_set(error, -ret,
2534                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2535                                            NULL, "enic: devcmd(action-alloc)");
2536                         goto error_with_ah;
2537                 }
2538                 ah->handle = args[0];
2539                 ret = rte_hash_add_key_data(fm->action_hash,
2540                                             (const void *)action_in,
2541                                             (void *)ah);
2542                 if (ret != 0) {
2543                         rte_flow_error_set(error, -ret,
2544                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2545                                            NULL,
2546                                            "enic: rte_hash_add_key_data(actn)");
2547                         goto error_with_action_handle;
2548                 }
2549                 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
2550                             ah->handle);
2551         }
2552
2553         /* Action handle struct is valid, increment reference count. */
2554         ah->ref++;
2555         *ah_o = ah;
2556         return 0;
2557 error_with_action_handle:
2558         args[0] = FM_ACTION_FREE;
2559         args[1] = ah->handle;
2560         ret = flowman_cmd(fm, args, 2);
2561         if (ret != 0)
2562                 rte_flow_error_set(error, -ret,
2563                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2564                                    NULL, "enic: devcmd(action-free)");
2565 error_with_ah:
2566         free(ah);
2567         return ret;
2568 }
2569
2570 /* Push match-action to the NIC. */
2571 static int
2572 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2573                          struct enic_fm_flow *fm_flow,
2574                          struct fm_tcam_match_entry *match_in,
2575                          struct fm_action *action_in,
2576                          uint32_t group,
2577                          uint8_t ingress,
2578                          struct rte_flow_error *error)
2579 {
2580         struct enic_fm_counter *ctr;
2581         struct enic_fm_action *ah = NULL;
2582         uint64_t entry_h;
2583         int ret;
2584
2585         ENICPMD_FUNC_TRACE();
2586
2587         /* Get or create an action handle. */
2588         ret = enic_action_handle_get(fm, action_in, error, &ah);
2589         if (ret)
2590                 return ret;
2591         match_in->ftm_action = ah->handle;
2592         fm_flow->action = ah;
2593
2594         /* Allocate counter if requested. */
2595         if (match_in->ftm_flags & FMEF_COUNTER) {
2596                 ret = enic_fm_counter_alloc(fm, error, &ctr);
2597                 if (ret) /* error has been filled in */
2598                         return ret;
2599                 fm_flow->counter_valid = true;
2600                 fm_flow->counter = ctr;
2601                 match_in->ftm_counter = ctr->handle;
2602         }
2603
2604         /*
2605          * Get the group's table (either TCAM or exact match table) and
2606          * add entry to it. If we use the exact match table, the handler
2607          * will translate the TCAM entry (match_in) to the appropriate
2608          * exact match entry and use that instead.
2609          */
2610         entry_h = FM_INVALID_HANDLE;
2611         if (group == FM_TCAM_RTE_GROUP) {
2612                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2613                                              error);
2614                 if (ret)
2615                         return ret;
2616                 /* Jump action might have a ref to fet */
2617                 fm_flow->fet = fm->fet;
2618                 fm->fet = NULL;
2619         } else {
2620                 struct enic_fm_fet *fet = NULL;
2621
2622                 ret = enic_fet_get(fm, group, ingress,
2623                                    &match_in->ftm_mask, &fet, error);
2624                 if (ret)
2625                         return ret;
2626                 fm_flow->fet = fet;
2627                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2628                                               error);
2629                 if (ret)
2630                         return ret;
2631         }
2632         /* Clear counter after adding entry, as it requires in-use counter */
2633         if (fm_flow->counter_valid) {
2634                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2635                 if (ret)
2636                         return ret;
2637         }
2638         fm_flow->entry_handle = entry_h;
2639         return 0;
2640 }
2641
2642 /* Push match-action to the NIC. */
2643 static struct rte_flow *
2644 enic_fm_flow_add_entry(struct enic_flowman *fm,
2645                        struct fm_tcam_match_entry *match_in,
2646                        struct fm_action *action_in,
2647                        const struct rte_flow_attr *attrs,
2648                        struct rte_flow_error *error)
2649 {
2650         struct enic_fm_flow *fm_flow;
2651         struct rte_flow *flow;
2652
2653         ENICPMD_FUNC_TRACE();
2654         match_in->ftm_position = attrs->priority;
2655         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2656         flow = calloc(1, sizeof(*flow));
2657         fm_flow = calloc(1, sizeof(*fm_flow));
2658         if (flow == NULL || fm_flow == NULL) {
2659                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2660                         NULL, "enic: cannot allocate rte_flow");
2661                 free(flow);
2662                 free(fm_flow);
2663                 return NULL;
2664         }
2665         flow->fm = fm_flow;
2666         fm_flow->action = NULL;
2667         fm_flow->entry_handle = FM_INVALID_HANDLE;
2668         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2669                                      attrs->group, attrs->ingress, error)) {
2670                 enic_fm_flow_free(fm, flow);
2671                 return NULL;
2672         }
2673         return flow;
2674 }
2675
2676 static void
2677 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2678                    struct rte_flow_error *error)
2679 {
2680         struct enic_fm_flow *fm_flow;
2681         struct enic_fm_jump_flow *j;
2682         struct fm_action *fma;
2683         uint32_t group;
2684
2685         ENICPMD_FUNC_TRACE();
2686         /*
2687          * Find the saved flows that should jump to the new table (fet).
2688          * Then delete the old TCAM entry that jumps to the default table,
2689          * and add a new one that jumps to the new table.
2690          */
2691         group = fet->group;
2692         j = find_jump_flow(fm, group);
2693         while (j) {
2694                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2695                             j->flow, group);
2696                 /* Delete old entry */
2697                 fm_flow = j->flow->fm;
2698                 __enic_fm_flow_free(fm, fm_flow);
2699
2700                 /* Add new entry */
2701                 fma = &j->action;
2702                 fma->fma_action_ops[0].exact.handle = fet->handle;
2703                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2704                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2705                         /* Cannot roll back changes at the moment */
2706                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2707                                     j->flow);
2708                 } else {
2709                         fm_flow->fet = fet;
2710                         fet->ref++;
2711                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2712                                     fet->group, fet->ref);
2713                 }
2714
2715                 TAILQ_REMOVE(&fm->jump_list, j, list);
2716                 free(j);
2717                 j = find_jump_flow(fm, group);
2718         }
2719 }
2720
2721 static int
2722 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2723                   struct rte_flow_error *error)
2724 {
2725         struct fm_tcam_match_entry *fm_tcam_entry;
2726         struct enic_fm_flow *fm_flow;
2727         struct fm_action *fm_action;
2728         struct fm_action_op fm_op;
2729         int ret;
2730
2731         ENICPMD_FUNC_TRACE();
2732         fm_flow = calloc(1, sizeof(*fm_flow));
2733         if (fm_flow == NULL) {
2734                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2735                         NULL, "enic: cannot allocate rte_flow");
2736                 return -ENOMEM;
2737         }
2738         /* Original egress hairpin flow */
2739         fm_tcam_entry = &fm->tcam_entry;
2740         fm_action = &fm->action;
2741         /* Use the match pattern of the egress flow as is, without counters */
2742         fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2743         /* The only action is steer to vnic */
2744         fm->action_op_count = 0;
2745         memset(fm_action, 0, sizeof(*fm_action));
2746         memset(&fm_op, 0, sizeof(fm_op));
2747         /* Always to queue 0 for now */
2748         fm_op.fa_op = FMOP_RQ_STEER;
2749         fm_op.rq_steer.rq_index = 0;
2750         fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2751         ret = enic_fm_append_action_op(fm, &fm_op, error);
2752         if (ret)
2753                 goto error_with_flow;
2754         ENICPMD_LOG(DEBUG, "add steer op");
2755         /* Add required END */
2756         memset(&fm_op, 0, sizeof(fm_op));
2757         fm_op.fa_op = FMOP_END;
2758         ret = enic_fm_append_action_op(fm, &fm_op, error);
2759         if (ret)
2760                 goto error_with_flow;
2761         /* Add the ingress flow */
2762         fm_flow->action = NULL;
2763         fm_flow->entry_handle = FM_INVALID_HANDLE;
2764         ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2765                                        FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2766         if (ret) {
2767                 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2768                 goto error_with_flow;
2769         }
2770         /* The new flow is now the egress flow's paired flow */
2771         flow->fm->hairpin_steer_flow = fm_flow;
2772         return 0;
2773
2774 error_with_flow:
2775         free(fm_flow);
2776         return ret;
2777 }
2778
2779 static void
2780 enic_fm_open_scratch(struct enic_flowman *fm)
2781 {
2782         fm->action_op_count = 0;
2783         fm->fet = NULL;
2784         fm->need_hairpin_steer = 0;
2785         fm->hairpin_steer_vnic_h = 0;
2786         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2787         memset(&fm->action, 0, sizeof(fm->action));
2788 }
2789
2790 static void
2791 enic_fm_close_scratch(struct enic_flowman *fm)
2792 {
2793         if (fm->fet) {
2794                 enic_fet_put(fm, fm->fet);
2795                 fm->fet = NULL;
2796         }
2797         fm->action_op_count = 0;
2798 }
2799
2800 static int
2801 enic_fm_flow_validate(struct rte_eth_dev *dev,
2802                       const struct rte_flow_attr *attrs,
2803                       const struct rte_flow_item pattern[],
2804                       const struct rte_flow_action actions[],
2805                       struct rte_flow_error *error)
2806 {
2807         struct fm_tcam_match_entry *fm_tcam_entry;
2808         struct fm_action *fm_action;
2809         struct enic_flowman *fm;
2810         int ret;
2811
2812         ENICPMD_FUNC_TRACE();
2813         fm = begin_fm(pmd_priv(dev));
2814         if (fm == NULL)
2815                 return -ENOTSUP;
2816         enic_fm_open_scratch(fm);
2817         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2818         if (!ret) {
2819                 fm_tcam_entry = &fm->tcam_entry;
2820                 fm_action = &fm->action;
2821                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2822                                         attrs->ingress);
2823         }
2824         enic_fm_close_scratch(fm);
2825         end_fm(fm);
2826         return ret;
2827 }
2828
2829 static int
2830 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2831                          struct rte_flow *flow, void *data,
2832                          struct rte_flow_error *error)
2833 {
2834         struct rte_flow_query_count *query;
2835         struct enic_fm_flow *fm_flow;
2836         struct enic_flowman *fm;
2837         uint64_t args[3];
2838         int rc;
2839
2840         ENICPMD_FUNC_TRACE();
2841         fm = begin_fm(pmd_priv(dev));
2842         query = data;
2843         fm_flow = flow->fm;
2844         if (!fm_flow->counter_valid) {
2845                 rc = rte_flow_error_set(error, ENOTSUP,
2846                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2847                         "enic: flow does not have counter");
2848                 goto exit;
2849         }
2850
2851         args[0] = FM_COUNTER_QUERY;
2852         args[1] = fm_flow->counter->handle;
2853         args[2] = query->reset;
2854         rc = flowman_cmd(fm, args, 3);
2855         if (rc) {
2856                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2857                             rc, fm_flow->counter->handle);
2858                 goto exit;
2859         }
2860         query->hits_set = 1;
2861         query->hits = args[0];
2862         query->bytes_set = 1;
2863         query->bytes = args[1];
2864         rc = 0;
2865 exit:
2866         end_fm(fm);
2867         return rc;
2868 }
2869
2870 static int
2871 enic_fm_flow_query(struct rte_eth_dev *dev,
2872                    struct rte_flow *flow,
2873                    const struct rte_flow_action *actions,
2874                    void *data,
2875                    struct rte_flow_error *error)
2876 {
2877         int ret = 0;
2878
2879         ENICPMD_FUNC_TRACE();
2880         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2881                 switch (actions->type) {
2882                 case RTE_FLOW_ACTION_TYPE_VOID:
2883                         break;
2884                 case RTE_FLOW_ACTION_TYPE_COUNT:
2885                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2886                         break;
2887                 default:
2888                         return rte_flow_error_set(error, ENOTSUP,
2889                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2890                                                   actions,
2891                                                   "action not supported");
2892                 }
2893                 if (ret < 0)
2894                         return ret;
2895         }
2896         return 0;
2897 }
2898
2899 static struct rte_flow *
2900 enic_fm_flow_create(struct rte_eth_dev *dev,
2901                     const struct rte_flow_attr *attrs,
2902                     const struct rte_flow_item pattern[],
2903                     const struct rte_flow_action actions[],
2904                     struct rte_flow_error *error)
2905 {
2906         struct fm_tcam_match_entry *fm_tcam_entry;
2907         struct fm_action *fm_action;
2908         struct enic_flowman *fm;
2909         struct enic_fm_fet *fet;
2910         struct rte_flow *flow;
2911         struct enic *enic;
2912         int ret;
2913
2914         ENICPMD_FUNC_TRACE();
2915         enic = pmd_priv(dev);
2916         fm = begin_fm(enic);
2917         if (fm == NULL) {
2918                 rte_flow_error_set(error, ENOTSUP,
2919                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2920                         "flowman is not initialized");
2921                 return NULL;
2922         }
2923         enic_fm_open_scratch(fm);
2924         flow = NULL;
2925         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2926         if (ret < 0)
2927                 goto error_with_scratch;
2928         fm_tcam_entry = &fm->tcam_entry;
2929         fm_action = &fm->action;
2930         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2931                                       attrs, error);
2932         if (flow) {
2933                 /* Add ingress rule that pairs with hairpin rule */
2934                 if (fm->need_hairpin_steer) {
2935                         ret = add_hairpin_steer(fm, flow, error);
2936                         if (ret) {
2937                                 enic_fm_flow_free(fm, flow);
2938                                 flow = NULL;
2939                                 goto error_with_scratch;
2940                         }
2941                 }
2942                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2943                 fet = flow->fm->fet;
2944                 if (fet && fet->default_key) {
2945                         /*
2946                          * Jump to non-existent group? Save the relevant info
2947                          * so we can convert this flow when that group
2948                          * materializes.
2949                          */
2950                         save_jump_flow(fm, flow, fet->group,
2951                                        fm_tcam_entry, fm_action);
2952                 } else if (fet && fet->ref == 1) {
2953                         /*
2954                          * A new table is created. Convert the saved flows
2955                          * that should jump to this group.
2956                          */
2957                         convert_jump_flows(fm, fet, error);
2958                 }
2959         }
2960
2961 error_with_scratch:
2962         enic_fm_close_scratch(fm);
2963         end_fm(fm);
2964         return flow;
2965 }
2966
2967 static int
2968 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2969                      __rte_unused struct rte_flow_error *error)
2970 {
2971         struct enic *enic = pmd_priv(dev);
2972         struct enic_flowman *fm;
2973
2974         ENICPMD_FUNC_TRACE();
2975         fm = begin_fm(enic);
2976         if (fm == NULL)
2977                 return 0;
2978         LIST_REMOVE(flow, next);
2979         enic_fm_flow_free(fm, flow);
2980         end_fm(fm);
2981         return 0;
2982 }
2983
2984 static int
2985 enic_fm_flow_flush(struct rte_eth_dev *dev,
2986                    __rte_unused struct rte_flow_error *error)
2987 {
2988         LIST_HEAD(enic_flows, rte_flow) internal;
2989         struct enic_fm_flow *fm_flow;
2990         struct enic_flowman *fm;
2991         struct rte_flow *flow;
2992         struct enic *enic = pmd_priv(dev);
2993
2994         ENICPMD_FUNC_TRACE();
2995
2996         fm = begin_fm(enic);
2997         if (fm == NULL)
2998                 return 0;
2999         /* Destroy all non-internal flows */
3000         LIST_INIT(&internal);
3001         while (!LIST_EMPTY(&enic->flows)) {
3002                 flow = LIST_FIRST(&enic->flows);
3003                 fm_flow = flow->fm;
3004                 LIST_REMOVE(flow, next);
3005                 if (flow->internal) {
3006                         LIST_INSERT_HEAD(&internal, flow, next);
3007                         continue;
3008                 }
3009                 /*
3010                  * If tables are null, then vNIC is closing, and the firmware
3011                  * has already cleaned up flowman state. So do not try to free
3012                  * resources, as it only causes errors.
3013                  */
3014                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
3015                         fm_flow->entry_handle = FM_INVALID_HANDLE;
3016                         fm_flow->action = NULL;
3017                         fm_flow->fet = NULL;
3018                 }
3019                 enic_fm_flow_free(fm, flow);
3020         }
3021         while (!LIST_EMPTY(&internal)) {
3022                 flow = LIST_FIRST(&internal);
3023                 LIST_REMOVE(flow, next);
3024                 LIST_INSERT_HEAD(&enic->flows, flow, next);
3025         }
3026         end_fm(fm);
3027         return 0;
3028 }
3029
3030 static int
3031 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
3032 {
3033         uint64_t args[2];
3034         int rc;
3035
3036         args[0] = FM_MATCH_TABLE_FREE;
3037         args[1] = handle;
3038         rc = flowman_cmd(fm, args, 2);
3039         if (rc)
3040                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
3041                             rc, handle);
3042         return rc;
3043 }
3044
3045 static int
3046 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
3047                         uint32_t max_entries, uint64_t *handle)
3048 {
3049         struct fm_tcam_match_table *tcam_tbl;
3050         uint64_t args[2];
3051         int rc;
3052
3053         ENICPMD_FUNC_TRACE();
3054         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
3055         tcam_tbl->ftt_direction = direction;
3056         tcam_tbl->ftt_stage = FM_STAGE_LAST;
3057         tcam_tbl->ftt_max_entries = max_entries;
3058         args[0] = FM_TCAM_TABLE_ALLOC;
3059         args[1] = fm->cmd.pa;
3060         rc = flowman_cmd(fm, args, 2);
3061         if (rc) {
3062                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
3063                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
3064                 return rc;
3065         }
3066         *handle = args[0];
3067         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
3068                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
3069         return 0;
3070 }
3071
3072 static int
3073 enic_fm_init_actions(struct enic_flowman *fm)
3074 {
3075         struct rte_hash *a_hash;
3076         char name[RTE_HASH_NAMESIZE];
3077         struct rte_hash_parameters params = {
3078                 .entries = FM_MAX_ACTION_TABLE_SIZE,
3079                 .key_len = sizeof(struct fm_action),
3080                 .hash_func = rte_jhash,
3081                 .hash_func_init_val = 0,
3082                 .socket_id = rte_socket_id(),
3083         };
3084
3085         ENICPMD_FUNC_TRACE();
3086         snprintf((char *)name, sizeof(name), "fm-ah-%s",
3087                  fm->owner_enic->bdf_name);
3088         params.name = name;
3089
3090         a_hash = rte_hash_create(&params);
3091         if (a_hash == NULL)
3092                 return -rte_errno;
3093         fm->action_hash = a_hash;
3094         return 0;
3095 }
3096
3097 static int
3098 enic_fm_init_counters(struct enic_flowman *fm)
3099 {
3100         ENICPMD_FUNC_TRACE();
3101         SLIST_INIT(&fm->counters);
3102         return enic_fm_more_counters(fm);
3103 }
3104
3105 static void
3106 enic_fm_free_all_counters(struct enic_flowman *fm)
3107 {
3108         uint64_t args[2];
3109         int rc;
3110
3111         args[0] = FM_COUNTER_BRK;
3112         args[1] = 0;
3113         rc = flowman_cmd(fm, args, 2);
3114         if (rc != 0)
3115                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
3116         rte_free(fm->counter_stack);
3117 }
3118
3119 static int
3120 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
3121 {
3122         int rc;
3123
3124         ENICPMD_FUNC_TRACE();
3125         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
3126                                     &fm->ig_tcam_hndl);
3127         if (rc)
3128                 return rc;
3129         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
3130                                     &fm->eg_tcam_hndl);
3131         return rc;
3132 }
3133
3134 static void
3135 enic_fm_free_tcam_tables(struct enic_flowman *fm)
3136 {
3137         ENICPMD_FUNC_TRACE();
3138         if (fm->ig_tcam_hndl) {
3139                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
3140                             fm->ig_tcam_hndl);
3141                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
3142                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
3143         }
3144         if (fm->eg_tcam_hndl) {
3145                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
3146                             fm->eg_tcam_hndl);
3147                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
3148                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
3149         }
3150 }
3151
3152 int
3153 enic_fm_init(struct enic *enic)
3154 {
3155         const struct rte_pci_addr *addr;
3156         struct enic_flowman *fm;
3157         uint8_t name[RTE_MEMZONE_NAMESIZE];
3158         int rc;
3159
3160         if (enic->flow_filter_mode != FILTER_FLOWMAN)
3161                 return 0;
3162         ENICPMD_FUNC_TRACE();
3163         /* Get vnic handle and save for port-id action */
3164         if (enic_is_vf_rep(enic))
3165                 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
3166         else
3167                 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
3168         rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
3169         if (rc) {
3170                 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
3171                             addr->bus, addr->devid, addr->function);
3172                 return rc;
3173         }
3174         /* Save UIF for egport action */
3175         enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
3176         ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
3177         /* Nothing else to do for representor. It will share the PF flowman */
3178         if (enic_is_vf_rep(enic))
3179                 return 0;
3180         fm = calloc(1, sizeof(*fm));
3181         if (fm == NULL) {
3182                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
3183                 return -ENOMEM;
3184         }
3185         fm->owner_enic = enic;
3186         rte_spinlock_init(&fm->lock);
3187         TAILQ_INIT(&fm->fet_list);
3188         TAILQ_INIT(&fm->jump_list);
3189         /* Allocate host memory for flowman commands */
3190         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
3191         fm->cmd.va = enic_alloc_consistent(enic,
3192                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
3193         if (!fm->cmd.va) {
3194                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
3195                 rc = -ENOMEM;
3196                 goto error_fm;
3197         }
3198         /* Allocate TCAM tables upfront as they are the main tables */
3199         rc = enic_fm_alloc_tcam_tables(fm);
3200         if (rc) {
3201                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
3202                 goto error_cmd;
3203         }
3204         /* Then a number of counters */
3205         rc = enic_fm_init_counters(fm);
3206         if (rc) {
3207                 ENICPMD_LOG(ERR, "cannot alloc counters");
3208                 goto error_tables;
3209         }
3210         /* set up action handle hash */
3211         rc = enic_fm_init_actions(fm);
3212         if (rc) {
3213                 ENICPMD_LOG(ERR, "cannot create action hash, error:%d", rc);
3214                 goto error_counters;
3215         }
3216         /*
3217          * One default exact match table for each direction. We hold onto
3218          * it until close.
3219          */
3220         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
3221         if (rc) {
3222                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
3223                 goto error_actions;
3224         }
3225         fm->default_ig_fet->ref = 1;
3226         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
3227         if (rc) {
3228                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
3229                 goto error_ig_fet;
3230         }
3231         fm->default_eg_fet->ref = 1;
3232         fm->vf_rep_tag = FM_VF_REP_TAG;
3233         enic->fm = fm;
3234         return 0;
3235
3236 error_ig_fet:
3237         enic_fet_free(fm, fm->default_ig_fet);
3238 error_actions:
3239         rte_hash_free(fm->action_hash);
3240 error_counters:
3241         enic_fm_free_all_counters(fm);
3242 error_tables:
3243         enic_fm_free_tcam_tables(fm);
3244 error_cmd:
3245         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3246                 fm->cmd.va, fm->cmd.pa);
3247 error_fm:
3248         free(fm);
3249         return rc;
3250 }
3251
3252 void
3253 enic_fm_destroy(struct enic *enic)
3254 {
3255         struct enic_flowman *fm;
3256         struct enic_fm_fet *fet;
3257
3258         ENICPMD_FUNC_TRACE();
3259         if (enic_is_vf_rep(enic)) {
3260                 delete_rep_flows(enic);
3261                 return;
3262         }
3263         if (enic->fm == NULL)
3264                 return;
3265         fm = enic->fm;
3266         enic_fm_flow_flush(enic->rte_dev, NULL);
3267         enic_fet_free(fm, fm->default_eg_fet);
3268         enic_fet_free(fm, fm->default_ig_fet);
3269         /* Free all exact match tables still open */
3270         while (!TAILQ_EMPTY(&fm->fet_list)) {
3271                 fet = TAILQ_FIRST(&fm->fet_list);
3272                 enic_fet_free(fm, fet);
3273         }
3274         enic_fm_free_tcam_tables(fm);
3275         enic_fm_free_all_counters(fm);
3276         rte_hash_free(fm->action_hash);
3277         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3278                 fm->cmd.va, fm->cmd.pa);
3279         fm->cmd.va = NULL;
3280         free(fm);
3281         enic->fm = NULL;
3282 }
3283
3284 int
3285 enic_fm_allocate_switch_domain(struct enic *pf)
3286 {
3287         const struct rte_pci_addr *cur_a, *prev_a;
3288         struct rte_eth_dev *dev;
3289         struct enic *cur, *prev;
3290         uint16_t domain_id;
3291         uint64_t vnic_h;
3292         uint16_t pid;
3293         int ret;
3294
3295         ENICPMD_FUNC_TRACE();
3296         if (enic_is_vf_rep(pf))
3297                 return -EINVAL;
3298         cur = pf;
3299         cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
3300         /* Go through ports and find another PF that is on the same adapter */
3301         RTE_ETH_FOREACH_DEV(pid) {
3302                 dev = &rte_eth_devices[pid];
3303                 if (!dev_is_enic(dev))
3304                         continue;
3305                 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
3306                         continue;
3307                 if (dev == cur->rte_dev)
3308                         continue;
3309                 /* dev is another PF. Is it on the same adapter? */
3310                 prev = pmd_priv(dev);
3311                 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
3312                 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
3313                         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
3314                                 cur->rte_dev->data->port_id,
3315                                 cur_a->bus, cur_a->devid, cur_a->function,
3316                                 dev->data->port_id,
3317                                 prev_a->bus, prev_a->devid, prev_a->function,
3318                                 prev->switch_domain_id);
3319                         cur->switch_domain_id = prev->switch_domain_id;
3320                         return 0;
3321                 }
3322         }
3323         ret = rte_eth_switch_domain_alloc(&domain_id);
3324         if (ret) {
3325                 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
3326                             ret);
3327         }
3328         cur->switch_domain_id = domain_id;
3329         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
3330                     cur->rte_dev->data->port_id,
3331                     cur_a->bus, cur_a->devid, cur_a->function,
3332                     domain_id);
3333         return ret;
3334 }
3335
3336 const struct rte_flow_ops enic_fm_flow_ops = {
3337         .validate = enic_fm_flow_validate,
3338         .create = enic_fm_flow_create,
3339         .destroy = enic_fm_flow_destroy,
3340         .flush = enic_fm_flow_flush,
3341         .query = enic_fm_flow_query,
3342 };
3343
3344 /* Add a high priority flow that loops representor packets to VF */
3345 int
3346 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
3347 {
3348         struct fm_tcam_match_entry *fm_tcam_entry;
3349         struct rte_flow *flow0, *flow1;
3350         struct fm_action *fm_action;
3351         struct rte_flow_error error;
3352         struct rte_flow_attr attrs;
3353         struct fm_action_op fm_op;
3354         struct enic_flowman *fm;
3355         struct enic *pf;
3356         uint8_t tag;
3357
3358         pf = vf->pf;
3359         fm = pf->fm;
3360         tag = fm->vf_rep_tag;
3361         enic_fm_open_scratch(fm);
3362         fm_tcam_entry = &fm->tcam_entry;
3363         fm_action = &fm->action;
3364         /* Egress rule: match WQ ID and tag+hairpin */
3365         fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
3366         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3367         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3368         memset(&fm_op, 0, sizeof(fm_op));
3369         fm_op.fa_op = FMOP_TAG;
3370         fm_op.tag.tag = tag;
3371         enic_fm_append_action_op(fm, &fm_op, &error);
3372         memset(&fm_op, 0, sizeof(fm_op));
3373         fm_op.fa_op = FMOP_EG_HAIRPIN;
3374         enic_fm_append_action_op(fm, &fm_op, &error);
3375         memset(&fm_op, 0, sizeof(fm_op));
3376         fm_op.fa_op = FMOP_END;
3377         enic_fm_append_action_op(fm, &fm_op, &error);
3378         attrs.group = 0;
3379         attrs.ingress = 0;
3380         attrs.egress = 1;
3381         attrs.priority = FM_HIGHEST_PRIORITY;
3382         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3383                                        &attrs, &error);
3384         enic_fm_close_scratch(fm);
3385         if (flow0 == NULL) {
3386                 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
3387                 return -EINVAL;
3388         }
3389         LIST_INSERT_HEAD(&pf->flows, flow0, next);
3390         /* Make this flow internal, so the user app cannot delete it */
3391         flow0->internal = 1;
3392         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
3393                     vf->vf_id, vf->pf_wq_idx, tag);
3394
3395         /* Ingress: steer hairpinned to VF RQ 0 */
3396         enic_fm_open_scratch(fm);
3397         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3398         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3399         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3400         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3401         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3402         memset(&fm_op, 0, sizeof(fm_op));
3403         fm_op.fa_op = FMOP_RQ_STEER;
3404         fm_op.rq_steer.rq_index = 0;
3405         fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
3406         enic_fm_append_action_op(fm, &fm_op, &error);
3407         memset(&fm_op, 0, sizeof(fm_op));
3408         fm_op.fa_op = FMOP_END;
3409         enic_fm_append_action_op(fm, &fm_op, &error);
3410         attrs.group = 0;
3411         attrs.ingress = 1;
3412         attrs.egress = 0;
3413         attrs.priority = FM_HIGHEST_PRIORITY;
3414         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3415                                        &attrs, &error);
3416         enic_fm_close_scratch(fm);
3417         if (flow1 == NULL) {
3418                 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
3419                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3420                 return -EINVAL;
3421         }
3422         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3423         flow1->internal = 1;
3424         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
3425                     vf->vf_id, tag, fm_op.rq_steer.rq_index);
3426         vf->rep2vf_flow[0] = flow0;
3427         vf->rep2vf_flow[1] = flow1;
3428         /* Done with this tag, use a different one next time */
3429         fm->vf_rep_tag++;
3430         return 0;
3431 }
3432
3433 /*
3434  * Add a low priority flow that matches all packets from VF and loops them
3435  * back to the representor.
3436  */
3437 int
3438 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3439 {
3440         struct fm_tcam_match_entry *fm_tcam_entry;
3441         struct rte_flow *flow0, *flow1;
3442         struct fm_action *fm_action;
3443         struct rte_flow_error error;
3444         struct rte_flow_attr attrs;
3445         struct fm_action_op fm_op;
3446         struct enic_flowman *fm;
3447         struct enic *pf;
3448         uint8_t tag;
3449
3450         pf = vf->pf;
3451         fm = pf->fm;
3452         tag = fm->vf_rep_tag;
3453         enic_fm_open_scratch(fm);
3454         fm_tcam_entry = &fm->tcam_entry;
3455         fm_action = &fm->action;
3456         /* Egress rule: match-any and tag+hairpin */
3457         fm_tcam_entry->ftm_data.fk_wq_id = 0;
3458         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3459         fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3460         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3461         memset(&fm_op, 0, sizeof(fm_op));
3462         fm_op.fa_op = FMOP_TAG;
3463         fm_op.tag.tag = tag;
3464         enic_fm_append_action_op(fm, &fm_op, &error);
3465         memset(&fm_op, 0, sizeof(fm_op));
3466         fm_op.fa_op = FMOP_EG_HAIRPIN;
3467         enic_fm_append_action_op(fm, &fm_op, &error);
3468         memset(&fm_op, 0, sizeof(fm_op));
3469         fm_op.fa_op = FMOP_END;
3470         enic_fm_append_action_op(fm, &fm_op, &error);
3471         attrs.group = 0;
3472         attrs.ingress = 0;
3473         attrs.egress = 1;
3474         attrs.priority = FM_LOWEST_PRIORITY;
3475         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3476                                        &attrs, &error);
3477         enic_fm_close_scratch(fm);
3478         if (flow0 == NULL) {
3479                 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3480                 return -EINVAL;
3481         }
3482         LIST_INSERT_HEAD(&pf->flows, flow0, next);
3483         /* Make this flow internal, so the user app cannot delete it */
3484         flow0->internal = 1;
3485         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3486                     vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3487
3488         /* Ingress: steer hairpinned to VF rep RQ */
3489         enic_fm_open_scratch(fm);
3490         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3491         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3492         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3493         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3494         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3495         memset(&fm_op, 0, sizeof(fm_op));
3496         fm_op.fa_op = FMOP_RQ_STEER;
3497         fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3498         fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3499         enic_fm_append_action_op(fm, &fm_op, &error);
3500         memset(&fm_op, 0, sizeof(fm_op));
3501         fm_op.fa_op = FMOP_END;
3502         enic_fm_append_action_op(fm, &fm_op, &error);
3503         attrs.group = 0;
3504         attrs.ingress = 1;
3505         attrs.egress = 0;
3506         attrs.priority = FM_HIGHEST_PRIORITY;
3507         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3508                                        &attrs, &error);
3509         enic_fm_close_scratch(fm);
3510         if (flow1 == NULL) {
3511                 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3512                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3513                 return -EINVAL;
3514         }
3515         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3516         flow1->internal = 1;
3517         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3518                     vf->vf_id, tag, vf->pf_rq_sop_idx);
3519         vf->vf2rep_flow[0] = flow0;
3520         vf->vf2rep_flow[1] = flow1;
3521         /* Done with this tag, use a different one next time */
3522         fm->vf_rep_tag++;
3523         return 0;
3524 }
3525
3526 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3527 static void
3528 delete_rep_flows(struct enic *enic)
3529 {
3530         struct enic_vf_representor *vf;
3531         struct rte_flow_error error;
3532         struct rte_eth_dev *dev;
3533         uint32_t i;
3534
3535         RTE_ASSERT(enic_is_vf_rep(enic));
3536         vf = VF_ENIC_TO_VF_REP(enic);
3537         dev = vf->pf->rte_dev;
3538         for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3539                 if (vf->vf2rep_flow[i])
3540                         enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3541         }
3542         for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3543                 if (vf->rep2vf_flow[i])
3544                         enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3545         }
3546 }
3547
3548 static struct enic_flowman *
3549 begin_fm(struct enic *enic)
3550 {
3551         struct enic_vf_representor *vf;
3552         struct enic_flowman *fm;
3553
3554         /* Representor uses PF flowman */
3555         if (enic_is_vf_rep(enic)) {
3556                 vf = VF_ENIC_TO_VF_REP(enic);
3557                 fm = vf->pf->fm;
3558         } else {
3559                 fm = enic->fm;
3560         }
3561         /* Save the API caller and lock if representors exist */
3562         if (fm) {
3563                 if (fm->owner_enic->switchdev_mode)
3564                         rte_spinlock_lock(&fm->lock);
3565                 fm->user_enic = enic;
3566         }
3567         return fm;
3568 }
3569
3570 static void
3571 end_fm(struct enic_flowman *fm)
3572 {
3573         fm->user_enic = NULL;
3574         if (fm->owner_enic->switchdev_mode)
3575                 rte_spinlock_unlock(&fm->lock);
3576 }